This notebook originally made by Matt Nicholl.
Interesting transients are selected from the ZTF stream, then these are looked up in various ways:
The Transient Name Server
PanSTARRS-1 Image Access
Open Astronomy Catalogs
At the end, you see the chosen candidates with light curves from ZTF and OAC, as well as the PanStarrs cutouts.

In [1]:
import mysql.connector
%matplotlib inline
import matplotlib.pyplot as plt
import numpy as np
import requests
import json
from collections import OrderedDict
import matplotlib.image as mpimg
import os
import csv
import io as StringIO
In [2]:
# connect to database. You should already have the file.
import settings
msl = mysql.connector.connect(\
            user    =settings.DB_USER, \
            password=settings.DB_PASS, \
            host    =settings.DB_HOST, \
In [3]:
import time, datetime
now =
print ("This notebook was run on " + now.isoformat())

jdnow = time.time()/86400 + 2440587.5
print ("and the Julian Date is {}".format(jdnow))
This notebook was run on 2019-06-17T09:02:34.898600
and the Julian Date is 2458651.835126143
In [4]:
# select good candidates
# can add further criteria...

# From the last 3 months
start_date = '%.1f' % (jdnow - 90)

# Default quality cuts
quality = 'rb >= 0.75 and nbad = 0 and fwhm <= 5 and elong <= 1.2 and abs(magdiff) <= 0.1 '

# Further cuts on individual detections
cand_of_interest = 'sgscore1 < 0.4 and jdstarthist > ' + start_date

# Cuts on object metadata
obj_of_interest = 'ncand >= 5 '

cursor = msl.cursor(buffered=True, dictionary=True)

# First query candidate database...
query_cand = 'SELECT *,candid,objectId from candidates WHERE '
query_cand += quality
query_cand += 'and '
query_cand += cand_of_interest
# ... making sure associated objects also pass cuts
query_cand += 'and objectId in (SELECT objectId from objects WHERE '
query_cand += obj_of_interest
query_cand += ')'

print ("Candidate query is:\n" + query_cand)

results = cursor.execute(query_cand)
n = cursor.rowcount
print ('\nfound %d good candidates' % n)
candidates = cursor.fetchall()
Candidate query is:
SELECT *,candid,objectId from candidates WHERE rb >= 0.75 and nbad = 0 and fwhm <= 5 and elong <= 1.2 and abs(magdiff) <= 0.1 and sgscore1 < 0.4 and jdstarthist > 2458561.8and objectId in (SELECT objectId from objects WHERE ncand >= 5 )

found 3533 good candidates
In [5]:
# Then query objects...
query_obj = 'SELECT * FROM objects WHERE '
query_obj += obj_of_interest
# ... making sure the candidates are the same set as above
query_obj += 'and objectId in (SELECT objectId from candidates WHERE '
query_obj += quality
query_obj += 'and '
query_obj += cand_of_interest
query_obj += ')'

print ("Object query is:\n" + query_cand)

results = cursor.execute(query_obj)
n = cursor.rowcount
print ('\nfound %d objects' % n)
objects = cursor.fetchall()

Object query is:
SELECT *,candid,objectId from candidates WHERE rb >= 0.75 and nbad = 0 and fwhm <= 5 and elong <= 1.2 and abs(magdiff) <= 0.1 and sgscore1 < 0.4 and jdstarthist > 2458561.8and objectId in (SELECT objectId from objects WHERE ncand >= 5 )

found 1091 objects
In [6]:
# Make dictionary of objects with light curves and metadata

# needed for light curve
lc_keys = ['jd', 'magpsf', 'sigmapsf', 'fid']

# attibutes of the nearest PS1 object
host_keys = ['distpsnr1', 'sgscore1', 'sgmag1', 'srmag1'] 

objectdict = {}

for row in objects:
    objectdict[row['objectId']] = row
    objectdict[row['objectId']]['lc'] = []
for j in candidates:
    lightcurvedict = dict((l, j[l]) for l in lc_keys if l in j)
    if 'host' not in objectdict[j['objectId']]:
        hostdict = dict((k, j[k]) for k in host_keys if k in j)
        objectdict[j['objectId']]['host'] = hostdict
In [7]:
# Apply further filters here

# If no filtering desired, simply set:
#objectdict = objectdict

# For this example, using objects with at least 10 detections
minpoints = 6

targetlist = []

for l in objectdict:
    if len(objectdict[l]['lc']) >= minpoints:
print('found %d events with >=%d detections' % (len(targetlist),minpoints))
found 163 events with >=6 detections

Query the Transient Name Server

In [8]:
# Query TNS for classifications

tns_url = ""

print('Searching TNS...')
print ("\nZTF objectId    TNS id    Type     Sherlock")
for i in targetlist:

    # start with cone search around ZTF coordinates
    search_obj = OrderedDict([
    search_url = tns_url+'search'
    search_data = [('api_key',(None, api_key)), ('data',(None,json.dumps(search_obj)))]

    r =, files=search_data)

    # If transient is known, will have an IAU name (AT/SN 20XXyy), add such names to our objects
    if r.json()['data']['reply']:
        iau_name = r.json()['data']['reply'][0]['objname']
        objectdict[i]['iau_name'] = r.json()['data']['reply'][0]['prefix'] + iau_name

        # Now check if object has a classification attached
        # Always will for 'SN' names, but could also happen for 'AT', e.g. if TDE rather than SN
        get_obj = OrderedDict([("objname",iau_name), ("photometry","0"), ("spectra","0")])
        get_url = tns_url+'object'
        get_data = [('api_key',(None, api_key)), ('data',(None,json.dumps(get_obj)))]
        r2 =, files=get_data)
        # If classified, add type to dictionary
        if r2.json()['data']['reply']['object_type']['name']:
            objectdict[i]['class'] = r2.json()['data']['reply']['object_type']['name']
            t = objectdict[i]['class']
            t = '     '

        print ("%s   %s   %s      %s" % (i, objectdict[i]['iau_name'], t,objectdict[i]['sherlock_classification']))

Searching TNS...

ZTF objectId    TNS id    Type     Sherlock
ZTF19aasekcx   SN2019ejp   SN IIn      SN
ZTF19aaraxhh   SN2019duf   SN Ia      NT
ZTF19aanesxt   SN2019cmh   SN Ia      SN
ZTF19aatgrxv   SN2019ens   SN Ia      NT
ZTF19aarioci   AT2019ehz              NT
ZTF19aanhlzn   SN2019cet   SN Ia      SN
ZTF19aarfkch   AT2019dwf              ORPHAN
ZTF19aasensj   AT2019ejr              NT
ZTF19aavjzny   AT2019gbp              SN
ZTF19aauitks   SN2019fcm   SN Ia      NT
ZTF19aauvblr   AT2019fza              SN
ZTF19aavhblr   SN2019fuo   SN II      NT
ZTF19aavipfj   AT2019gyv              SN
ZTF19aatvmbo   SN2019enp   SN Ia-91T-like      SN
ZTF19aavmlqh   SN2019gbq   SN Ia      NT
ZTF19aavlgdy   AT2019fyh              NT
ZTF19aavrswr   SN2019gcw   SN Ia      ORPHAN
ZTF19aavpzke   AT2019ghp              NT
ZTF19aavwbpc   AT2019gtl              NT
ZTF19aavqotz   AT2019ged              NT
ZTF19aavhwxs   SN2019gct   SN Ia      NT
ZTF19aawnqoj   SN2019gxg   SN Ia      NT
ZTF19aavowpa   AT2019gxp              NT
ZTF19aavlnqp   AT2019had              NT
ZTF19aaqxcfs   SN2019drk   SN Ia      SN
ZTF19aavwbyn   AT2019gei              SN
ZTF19aawfbtg   AT2019hge              SN
ZTF19aaozsuh   SN2019dde   SN IIn      SN
ZTF19aawolyt   AT2019grq   CV      SN
ZTF19aavocxo   AT2019gav              NT
ZTF19aaupkev   SN2019flf   SN Ia      SN
ZTF19aansyyn   SN2019cmz   SN Ia      SN
ZTF19aarjswp   AT2019duz              ORPHAN
ZTF19aaqcqkv   SN2019dpu   SN Ia      SN
ZTF19aanfyey   SN2019cdz   SN Ia      NT
ZTF19aasbphu   AT2019eqb              NT
ZTF19aaviiyy   AT2019gbj              ORPHAN
ZTF19aavoegl   AT2019gig              ORPHAN
ZTF19aanxosu   SN2019dat   SN Ia      NT
ZTF19aavkvpw   SN2019gcl   SN Ic-BL      SN
ZTF19aatubsj   AT2019fdr              NT
ZTF19aavvozj   AT2019gxt              SN
ZTF19aavbomb   AT2019fzb              SN
ZTF19aarnhko   AT2019edw              SN
ZTF19aaobwpx   SN2019czy   SN Ia      SN
ZTF19aaulzvc   SN2019ewz   SN Ia      UNCLEAR
ZTF19aauisdr   SN2019fci   SN II      NT
ZTF19aaoxijx   AT2019cxz              NT
ZTF19aanetjx   SN2019crd   SN Ia      NT
ZTF19aavktnq   AT2019gck              SN
ZTF19aavxsxd   AT2019gew              ORPHAN
ZTF19aauxmqj   SN2019fxs   SN Ia      SN
ZTF19aarjspm   AT2019dvb              SN
ZTF19aatgrpm   AT2019eod              SN
ZTF19aanxgle   AT2019das              SN
ZTF19aavouyw   SN2019gfm   SN Ic-BL      SN
ZTF19aawlluc   AT2019goa              NT
ZTF19aavnwtf   SN2019geb   SN Ia      SN
ZTF19aavkhuy   SN2019fpf   SN Ia      ORPHAN
ZTF19aarifvr   AT2019ecw              SN
ZTF19aaprhvf   AT2019ekr              VS
ZTF19aaoxvfe   SN2019dgz   SN Ib      SN
ZTF19aarixue   SN2019edu   SN Ia      SN
ZTF19aatlqdf   AT2019eju              SN
ZTF19aavjkwh   AT2019gbl              SN
ZTF19aatgznl   SN2019ekb   SN Ia      SN
ZTF19aanircs   SN2019cnb   SN Ia      SN
ZTF19aathapt   AT2019eom              SN
ZTF19aaxfcpq   AT2019gwc              SN
ZTF19aawyepw   AT2019hax              NT
ZTF19aavspux   AT2019hfk              NT
ZTF19aaupklw   AT2019fgx              NT
ZTF19aavwcow   AT2019ful              NT
ZTF19aaniore   SN2019ceg   SN II      SN
ZTF19aapreis   AT2019dsg   TDE      NT
ZTF19aawvisn   AT2019gzb              SN
ZTF19aaycomi   SN2019hbb   SN Ia      SN
ZTF19aavleuf   AT2019fyq              SN
ZTF19aanramr   SN2019cmj   SN Ia      SN
ZTF19aanetvz   AT2019cts              SN
ZTF19aavledx   AT2019fxq              NT
ZTF19aavqoyu   AT2019gef              SN
ZTF19aanlekq   SN2019cct   SN II      SN
ZTF19aanydko   SN2019coj   SN Ia      SN
ZTF19aaojljx   AT2019ehb              NT
ZTF19aapzbjr   AT2019dvv              SN
ZTF19aaqyvdn   SN2019eqk   SN Ia      ORPHAN
ZTF19aanmhlu   AT2019cjs              ORPHAN
ZTF19aavjlkx   AT2019gbm              SN
ZTF19aaugaam   SN2019evh   SN II      SN
ZTF19aaviinr   AT2019gun              UNCLEAR
ZTF19aavhypb   SN2019fyj   SN Ia      SN
ZTF19aawudku   AT2019gtp              SN
ZTF19aaoikti   SN2019cyy   SN Ia      SN
ZTF19aanmeyw   SN2019cjq   SN Ia      SN
ZTF19aawgfnn   SN2019gsh   SN Ia      NT
ZTF19aavounq   SN2019gel   SN Ia      SN
ZTF19aatlqju   AT2019eqd              SN
ZTF19aaxugwk   AT2019hau   CV      ORPHAN
ZTF19aaripqw   SN2019ecs   SN Ia      SN
ZTF19aarouww   AT2019fdk              SN
ZTF19aatgopg   AT2019eoc              NT
ZTF19aaurstb   SN2019gcg   SN Ia      SN
ZTF19aauishy   SN2019evl   SN II      NT
ZTF19aavxufb   SN2019ftg   SN Ia      SN
ZTF19aavsahh   SN2019giy   SN Ia      NT
ZTF19aavifuy   AT2019gbi              ORPHAN
ZTF19aavihif   AT2019hbh              SN
ZTF19aarinmx   AT2019duo              SN
ZTF19aavnzti   AT2019gsw              NT
ZTF19aavidqf   SN2019gbh   SN Ia-91T-like      SN
ZTF19aaxffum   SN2019gwa   SN Ia-91T-like      NT
ZTF19aapafcu   AT2019def              UNCLEAR
ZTF19aavhvkd   SN2019fma   SN Ia      SN
ZTF19aavkptg   SN2019gqs   SN IIP      SN
ZTF19aavqiuw   SN2019fzt   SN Ia      SN
ZTF19aaomkvl   SN2019crp   SN Ia      SN
ZTF19aardunx   SN2019dpw   SN Ia      SN
ZTF19aaurxsa   SN2019fcf   SN Ia      SN
ZTF19aaoyjoz   SN2019cxe   SN Ia      SN
ZTF19aauhhjy   AT2019fcq              NT
ZTF19aathllr   SN2019eji   SN II      SN
ZTF19aatmkll   SN2019fcb   SN Ia      SN
ZTF19aauxxgk   SN2019ffn   SN II      SN
ZTF19aavmgtv   SN2019gcm   SN Ia      SN
ZTF19aaxfzao   SN2019hgw   SN Ia      SN
ZTF19aaoyech   SN2019cya   SN Ia      SN
ZTF19aaugoig   AT2019evf              SN
ZTF19aavkqrg   AT2019ftk              SN
ZTF19aavoayk   AT2019gxl              NT
ZTF19aaprdox   AT2019dsf              AGN
ZTF19aaqdkrm   SN2019dod   SN II      SN
ZTF19aavhxby   SN2019fzm   SN Ia      SN
ZTF19aavbkly   SN2019fmv   SN IIP      SN
ZTF19aavqics   SN2019gxo   SN II      SN
ZTF19aatqzim   SN2019eoh   SN II      SN
ZTF19aaujiwc   SN2019evz   SN Ia      NT
ZTF19aarixve   AT2019evw              SN
ZTF19aaydpiq   AT2019hhc              SN
ZTF19aauqwna   SN2019fem   SN IIP      SN
ZTF19aapafqd   SN2019dvw   SN II      SN
ZTF19aaumats   AT2019euo              SN
ZTF19aatvlfl   AT2019elm              SN
ZTF19aarjfqe   SN2019dvd   SN II      SN
ZTF19aarxvtu   AT2019hae              ORPHAN
ZTF19aanxwrq   AT2019dsb              AGN
ZTF19aawgxdn   SN2019gmh   SN II      SN
ZTF19aatlmbo   SN2019ein   SN Ia      SN
ZTF19aanrrqu   SN2019clp   SN      SN
ZTF19aanhhal   SN2019cec   SN II      SN
ZTF19aanxgrz   SN2019cnu   SN Ia      SN
ZTF19aavotvg   AT2019gmf              ORPHAN

In [9]:
# Apply more cuts based on TNS data

targetlist2 = []

# e.g. classified transients only, excluding SNe Ia:
for i in targetlist:
    if 'class' in objectdict[i]:
        if objectdict[i]['class'] != 'SN Ia':
print('%d events passing final cuts' % len(targetlist2))
30 events passing final cuts
In [10]:
# Get PS1 image of field

ps1_url = ''

cutout_url = ''

print('Downloading PS1 cutouts...')

for i in targetlist2:

    dest_file = i + '.jpg'
    objectdict[i]['cutout'] = dest_file

    if not os.path.exists(dest_file):

        # first step searches for the ps1 images matching target coordinates
        ps1_search_url = ps1_url+'&ra='+str(objectdict[i]['ramean'])+'&dec='+str(objectdict[i]['decmean'])
        ps1_im = requests.get(ps1_search_url)
        #print("PS1 im.text = ")
        # consistent formatting, hardcoding positions of g,r,i image names within query result
        #ps1_g = ps1_im.text.split(' ')[15]
        #ps1_r = ps1_im.text.split(' ')[31]
        #ps1_i = ps1_im.text.split(' ')[23]

        # 2019-06-16 KWS Modified this to use a csv and StringIO object and present a dictionary.
        response = csv.DictReader(StringIO.StringIO(ps1_im.text), delimiter=' ', skipinitialspace = True)
        respList = []
        for row in response:
        ps1_g = respList[0]['filename']
        ps1_r = respList[2]['filename']
        ps1_i = respList[1]['filename']

        cutout_search_url = cutout_url + '&blue='+ps1_g + '&green='+ps1_r + '&red='+ps1_i
        cutout_search_url +=  '&ra='+str(objectdict[i]['ramean']) + '&dec='+str(objectdict[i]['decmean'])

        cmd = 'wget -O %s "%s"' % (dest_file, cutout_search_url)
Downloading PS1 cutouts...
ZTF19aasekcx 212.1191200851852 29.269773607407412
ZTF19aavhblr 232.93076554375 16.71369424375
ZTF19aatvmbo 320.55933567777777 5.788522255555556
ZTF19aaozsuh 217.05015997045456 -1.6041846795454544
ZTF19aawolyt 243.1838834090909 6.11314980909091
ZTF19aavkvpw 179.22925296363636 22.14207082727273
ZTF19aauisdr 201.95510837142857 27.72950377142857
ZTF19aavouyw 233.9441076909091 24.062490172727276
ZTF19aaoxvfe 184.01919780416665 29.851673104166665
ZTF19aaniore 246.96516813714288 62.69236173142856
ZTF19aapreis 314.26236382631583 14.204505531578945
ZTF19aanlekq 232.83778763333333 35.98404963030303
ZTF19aaugaam 200.07899513571425 8.180401671428571
ZTF19aaxugwk 292.093070125 55.533625725
ZTF19aauishy 202.755265875 34.153488749999994
ZTF19aavidqf 268.0652511275862 47.26780203103448
ZTF19aaxffum 239.67163309499998 11.240284695
ZTF19aavkptg 174.06575479166668 49.15349891666667
ZTF19aathllr 258.6084457681818 19.810947454545456
ZTF19aauxxgk 296.92462472666665 74.99833452666668
ZTF19aaqdkrm 201.45822850588237 34.49543664705883
ZTF19aavbkly 187.3908464642857 35.770052099999994
ZTF19aavqics 172.24341855454543 22.08958410909091
ZTF19aatqzim 195.955632205 38.28915851
ZTF19aauqwna 296.1921905500001 44.713646966666666
ZTF19aapafqd 239.94416924848483 37.03371098484848
ZTF19aarjfqe 187.77871220625 0.46516651875000004
ZTF19aawgxdn 247.76318915833335 41.153963250000004
ZTF19aanrrqu 183.41598925142856 16.123455231428572
ZTF19aanhhal 205.41977120540543 55.66965347837838

Get PanSTARRS image cutouts

Light curves from Open Astronomy Catalogs

In [11]:
# If known transient, get data from Open Astronomy Catalogs

oac_url = '' # object name goes here
oac_query_string = '/photometry/time+magnitude+e_magnitude+band?time&magnitude'

for i in targetlist2:
    if 'iau_name' in objectdict[i]:
        query_url = oac_url + objectdict[i]['iau_name'] + oac_query_string
        oac_phot = requests.get(query_url)
        d = oac_phot.json()
        objectname = objectdict[i]['iau_name']
        if objectname in d:
            objectdict[i]['oac'] = d[objectname]['photometry']
In [12]:
# Plot light curves

ztf_filts = {1:'g', 2:'r'}
ztf_markers = {1:'s', 2:'D'}

# if more filters from other surveys, add to colour pallet here:
other_filts = {'orange':'orange', 'cyan':'c', 'G':'darkkhaki', 'C':'pink',
               'w':'m', 'g':'g', 'r':'r', 'i':'goldenrod', 'z':'k'}

n = 1

for i in targetlist2:
    for j in objectdict[i]['lc']:
                     fmt=ztf_markers[j['fid']],color=ztf_filts[j['fid']],label='ZTF '+ztf_filts[j['fid']])

    # include photometry from open catalog:
    if 'oac' in objectdict[i]:
        for k in objectdict[i]['oac']:

    # construct legend
    handles, labels = plt.gca().get_legend_handles_labels()
    by_label = OrderedDict(zip(labels, handles))
    plt.legend(by_label.values(), by_label.keys())

    # format figure
    figtitle = i
    if 'iau_name' in objectdict[i]:
        figtitle += ' = '+objectdict[i]['iau_name']
    if 'class' in objectdict[i]:
        figtitle += ' ('+objectdict[i]['class']+')'
    # show cutout
    if 'cutout' in objectdict[i]:
        if os.path.exists(objectdict[i]['cutout']):
            file = objectdict[i]['cutout']
                img = mpimg.imread(file)
                print("cannot find file ", file)

    print('' % i)
    n += 1