This notebook originally made by Matt Nicholl.
Interesting transients are selected from the ZTF stream, then these are looked up in various ways:
The Transient Name Server
PanSTARRS-1 Image Access
Open Astronomy Catalogs
At the end, you see the chosen candidates with light curves from ZTF and OAC, as well as the PanStarrs cutouts.

In [84]:
import mysql.connector
%matplotlib inline
import matplotlib.pyplot as plt
import numpy as np
import requests
import json
from collections import OrderedDict
import matplotlib.image as mpimg
import os
In [85]:
# connect to database. You should already have the settings.py file.
import settings
msl = mysql.connector.connect(\
            user    =settings.DB_USER, \
            password=settings.DB_PASS, \
            host    =settings.DB_HOST, \
            database='ztf')
In [86]:
import time, datetime
now = datetime.datetime.now()
print ("This notebook was run on " + now.isoformat())

jdnow = time.time()/86400 + 2440587.5
print ("and the Julian Date is {}".format(jdnow))
This notebook was run on 2018-11-26T13:25:03.058293
and the Julian Date is 2458449.059063179
In [87]:
# select good candidates
# can add further criteria...

# From the last 3 months
start_date = '%.1f' % (jdnow - 90)

# Default quality cuts
quality = 'rb >= 0.75 and nbad = 0 and fwhm <= 5 and elong <= 1.2 and abs(magdiff) <= 0.1 '

# Further cuts on individual detections
cand_of_interest = 'sgscore1 < 0.4 and jdstarthist > ' + start_date

# Cuts on object metadata
obj_of_interest = 'ncand >= 5 '

cursor = msl.cursor(buffered=True, dictionary=True)

# First query candidate database...
query_cand = 'SELECT *,candid,objectId from candidates WHERE '
query_cand += quality
query_cand += 'and '
query_cand += cand_of_interest
# ... making sure associated objects also pass cuts
query_cand += 'and objectId in (SELECT objectId from objects WHERE '
query_cand += obj_of_interest
query_cand += ')'

print ("Candidate query is:\n" + query_cand)

results = cursor.execute(query_cand)
n = cursor.rowcount
print ('\nfound %d good candidates' % n)
candidates = cursor.fetchall()
Candidate query is:
SELECT *,candid,objectId from candidates WHERE rb >= 0.75 and nbad = 0 and fwhm <= 5 and elong <= 1.2 and abs(magdiff) <= 0.1 and sgscore1 < 0.4 and jdstarthist > 2458359.1and objectId in (SELECT objectId from objects WHERE ncand >= 5 )

found 1721 good candidates
In [88]:
# Then query objects...
query_obj = 'SELECT * FROM objects WHERE '
query_obj += obj_of_interest
# ... making sure the candidates are the same set as above
query_obj += 'and objectId in (SELECT objectId from candidates WHERE '
query_obj += quality
query_obj += 'and '
query_obj += cand_of_interest
query_obj += ')'

print ("Object query is:\n" + query_cand)

results = cursor.execute(query_obj)
n = cursor.rowcount
print ('\nfound %d objects' % n)
objects = cursor.fetchall()

cursor.close()
Object query is:
SELECT *,candid,objectId from candidates WHERE rb >= 0.75 and nbad = 0 and fwhm <= 5 and elong <= 1.2 and abs(magdiff) <= 0.1 and sgscore1 < 0.4 and jdstarthist > 2458359.1and objectId in (SELECT objectId from objects WHERE ncand >= 5 )

found 613 objects
Out[88]:
True
In [89]:
# Make dictionary of objects with light curves and metadata

# needed for light curve
lc_keys = ['jd', 'magpsf', 'sigmapsf', 'fid']

# attibutes of the nearest PS1 object
host_keys = ['distpsnr1', 'sgscore1', 'sgmag1', 'srmag1'] 

objectdict = {}

for row in objects:
    objectdict[row['objectId']] = row
    objectdict[row['objectId']]['lc'] = []
    
for j in candidates:
    lightcurvedict = dict((l, j[l]) for l in lc_keys if l in j)
    objectdict[j['objectId']]['lc'].append(lightcurvedict)
    if 'host' not in objectdict[j['objectId']]:
        hostdict = dict((k, j[k]) for k in host_keys if k in j)
        objectdict[j['objectId']]['host'] = hostdict
In [90]:
# Apply further filters here

# If no filtering desired, simply set:
#objectdict = objectdict

# For this example, using objects with at least 10 detections
minpoints = 6

targetlist = []

for l in objectdict:
    if len(objectdict[l]['lc']) >= minpoints:
        targetlist.append(l)
        
print('found %d events with >=%d detections' % (len(targetlist),minpoints))
found 70 events with >=6 detections

Query the Transient Name Server

In [91]:
# Query TNS for classifications

tns_url = "https://wis-tns.weizmann.ac.il/api/get/"
api_key="9a8a461eb820b886d28d412290223ad6dc971ebc"

print('Searching TNS...')
print ("\nZTF objectId    TNS id    Type     Sherlock")
for i in targetlist:

    # start with cone search around ZTF coordinates
    search_obj = OrderedDict([
        ("ra",objectdict[i]['ramean']), 
        ("dec",objectdict[i]['decmean']), 
        ("radius","2.5"), 
        ("units","arcsec"), 
        ("objname",""), 
        ("internal_name","")]) 
    search_url = tns_url+'search'
    search_data = [('api_key',(None, api_key)), ('data',(None,json.dumps(search_obj)))]

    r = requests.post(search_url, files=search_data)

    # If transient is known, will have an IAU name (AT/SN 20XXyy), add such names to our objects
    if r.json()['data']['reply']:
        iau_name = r.json()['data']['reply'][0]['objname']
        objectdict[i]['iau_name'] = r.json()['data']['reply'][0]['prefix'] + iau_name


        # Now check if object has a classification attached
        # Always will for 'SN' names, but could also happen for 'AT', e.g. if TDE rather than SN
        get_obj = OrderedDict([("objname",iau_name), ("photometry","0"), ("spectra","0")])
        get_url = tns_url+'object'
        get_data = [('api_key',(None, api_key)), ('data',(None,json.dumps(get_obj)))]
  
        r2 = requests.post(get_url, files=get_data)
     
        # If classified, add type to dictionary
        if r2.json()['data']['reply']['object_type']['name']:
            objectdict[i]['class'] = r2.json()['data']['reply']['object_type']['name']
            t = objectdict[i]['class']
        else:
            t = '     '


        print ("%s   %s   %s      %s" % (i, objectdict[i]['iau_name'], t,objectdict[i]['sherlock_classification']))

print('\nDone.')
Searching TNS...

ZTF objectId    TNS id    Type     Sherlock
ZTF18abuncpe   SN2018ggz   SN Ia      SN
ZTF18acdvvgx   AT2018ino              SN
ZTF18abtgdxy   AT2018fvz              SN
ZTF18absdgon   SN2018frx   SN Ia      SN
ZTF18abxftqm   AT2018hco              NT
ZTF18acbwarv   AT2018htu              SN
ZTF18abtmbaz   SN2018gff   SN Ia      NT
ZTF18absldfl   SN2018fsm   SN II      SN
ZTF18abuhyjv   SN2018gjt   SN Ia      NT
ZTF18abwkczr   SN2018grc   SN Ia      UNCLEAR
ZTF18abslpjy   SN2018ffj   SLSN-I      SN
ZTF18abshezu   SN2018gft   SLSN-I      ORPHAN
ZTF18abspnlz   AT2018cwy              NT
ZTF18acclexy   AT2018ikm              NT
ZTF18absbgqd   SN2018fvu   SN II      UNCLEAR
ZTF18abvfecb   SN2018gvm   SN II      SN
ZTF18abvgiug   AT2018vp              NT
ZTF18abvrvdk   AT2018ike              NT
ZTF18accndxn   AT2018igt              VS
ZTF18acbznkf   AT2018iau              NT
ZTF18abuvcdh   AT2018fqa              SN
ZTF18acbxhua   SN2018how   SN Ia      NT
ZTF18abukavn   SN2018gep   SN Ic-BL      NT
ZTF18abtqevs   SN2018grt   SN Ia      VS
ZTF18abtsuqv   SN2018ddi   SN Ia      NT
ZTF18abtswjk   SN2018gfx   SN IIn      NT
ZTF18acbwaax   SN2018hoj   SN Ia      SN
ZTF18absgnio   SN2018fuu   SN Ia      NT
ZTF18abtgnky   SN2018fey   SN Ia      SN
ZTF18abrlljc   SN2018fso   SN IIP      SN
ZTF18absoomk   SN2018ghp   SN Ia      ORPHAN
ZTF18abtlrqh   SN2018emo   SN Ia      NT
ZTF18abtywvz   AT2018euv              SN
ZTF18abthaii   AT2018fzp              SN
ZTF18abrqedj   SN2018fqz   SN Ia      NT
ZTF18acbxkop   SN2018hrq   SN II      NT
ZTF18abugmrg   AT2018ggy              NT
ZTF18accnnyu   SN2018hsy   SN Ia      NT
ZTF18abvfjwt   AT2018gsy              UNCLEAR
ZTF18abyfcns   AT2018gua              NT
ZTF18abxbmcl   SN2018hcp   SN IIP      SN
ZTF18absgnbm   SN2018fit   SN II      SN
ZTF18abtfrbf   SN2018gdg   SN Ia      SN
ZTF18abunigc   AT2018iev              AGN
ZTF18abrqfjs   AT2018fmq   CV      SN
ZTF18absyyig   SN2018fqk   SN Ia      NT
ZTF18abvdmfx   SN2018cvn   SN II      SN
ZTF18abucmwa   AT2018gga              VS
ZTF18abuhzfc   SN2018ggx   SN Ia      NT
ZTF18abuifwq   AT2018ibk              NT
ZTF18acbujhw   SN2018hrt   SN Ia      SN
ZTF18acehjtj   AT2018igz              CV

Done.
In [92]:
# Apply more cuts based on TNS data

targetlist2 = []

# e.g. classified transients only, excluding SNe Ia:
for i in targetlist:
    if 'class' in objectdict[i]:
        if objectdict[i]['class'] != 'SN Ia':
            targetlist2.append(i)
        
print('%d events passing final cuts' % len(targetlist2))
13 events passing final cuts

Get PanSTARRS image cutouts

In [93]:
# Get PS1 image of field

ps1_url = 'http://ps1images.stsci.edu/cgi-bin/ps1filenames.py?&filters=gri'

cutout_url = 'http://ps1images.stsci.edu/cgi-bin/fitscut.cgi?&filetypes=stack&size=240'

print('Downloading PS1 cutouts...')

for i in targetlist2:

    dest_file = i + '.jpg'
    objectdict[i]['cutout'] = dest_file

    if not os.path.exists(dest_file):

        # first step searches for the ps1 images matching target coordinates
        ps1_search_url = ps1_url+'&ra='+str(objectdict[i]['ramean'])+'&dec='+str(objectdict[i]['decmean'])
        #print(i,objectdict[i]['ramean],objectdict[i]['decmean'])
        #print(ps1_search_url)
        ps1_im = requests.get(ps1_search_url)
        print(ps1_im.text)
        
        # consistent formatting, hardcoding positions of g,r,i image names within query result
        ps1_g = ps1_im.text.split(' ')[15]
        ps1_r = ps1_im.text.split(' ')[31]
        ps1_i = ps1_im.text.split(' ')[23]
#        print(ps1_g,ps1_r,ps1_i)

        cutout_search_url = cutout_url + '&blue='+ps1_g + '&green='+ps1_r + '&red='+ps1_i
        cutout_search_url +=  '&ra='+str(objectdict[i]['ramean']) + '&dec='+str(objectdict[i]['decmean'])

        cmd = 'wget -O %s "%s"' % (dest_file, cutout_search_url)
#        print(cmd)
        os.system(cmd)
    
print('Done.')
Downloading PS1 cutouts...
Done.

Light curves from Open Astronomy Catalogs

In [94]:
# If known transient, get data from Open Astronomy Catalogs

oac_url = 'https://api.astrocats.space/' # object name goes here
oac_query_string = '/photometry/time+magnitude+e_magnitude+band?time&magnitude'

for i in targetlist2:
    if 'iau_name' in objectdict[i]:
        query_url = oac_url + objectdict[i]['iau_name'] + oac_query_string
        oac_phot = requests.get(query_url)
        objectdict[i]['oac'] = oac_phot.json()[objectdict[i]['iau_name']]['photometry']
In [95]:
# Plot light curves

ztf_filts = {1:'g', 2:'r'}
ztf_markers = {1:'s', 2:'D'}

# if more filters from other surveys, add to colour pallet here:
other_filts = {'orange':'orange', 'cyan':'c', 'G':'darkkhaki', 'C':'pink',
               'w':'m', 'g':'g', 'r':'r', 'i':'goldenrod', 'z':'k'}

n = 1

for i in targetlist2:
    plt.figure(n,(14,6))    
    plt.subplot(121)
    for j in objectdict[i]['lc']:
        plt.errorbar(j['jd']-2400000.5,j['magpsf'],j['sigmapsf'],markersize=6,
                     fmt=ztf_markers[j['fid']],color=ztf_filts[j['fid']],label='ZTF '+ztf_filts[j['fid']])

    # include photometry from open catalog:
    if 'oac' in objectdict[i]:
        for k in objectdict[i]['oac']:
            try:
                plt.errorbar(k[0],k[1],k[2],fmt='o',markersize=4,color=other_filts[k[3]],label=k[3])
            except:
                try:
                    plt.errorbar(k[0],k[1],fmt='o',markersize=4,color=other_filts[k[3]],label=k[3])
                except:
                    plt.errorbar(k[0],k[1],fmt='o',markersize=4,color='0.7',label='unknown')

    # construct legend
    handles, labels = plt.gca().get_legend_handles_labels()
    by_label = OrderedDict(zip(labels, handles))
    plt.legend(by_label.values(), by_label.keys())

    # format figure
    plt.xlabel('MJD')
    plt.ylabel('mag')
    plt.gca().invert_yaxis()
    figtitle = i
    if 'iau_name' in objectdict[i]:
        figtitle += ' = '+objectdict[i]['iau_name']
    if 'class' in objectdict[i]:
        figtitle += ' ('+objectdict[i]['class']+')'
    plt.title(figtitle)
    
    # show cutout
    if 'cutout' in objectdict[i]:
        if os.path.exists(objectdict[i]['cutout']):
            plt.subplot(122)
            img = mpimg.imread(objectdict[i]['cutout'])
            plt.imshow(img)

    print('https://lasair.roe.ac.uk/object/%s/' % i)
    plt.show()
    
    n += 1
    
print('Done.')
https://lasair.roe.ac.uk/object/ZTF18absldfl/
https://lasair.roe.ac.uk/object/ZTF18abslpjy/
https://lasair.roe.ac.uk/object/ZTF18abshezu/