Initial commit.

README.md

@@ -0,0 +1,11 @@
+
+### to install
+
+pip install --user django attrs
+git clone https://github.com/jsaponara/graphcast
+
+### to test
+
+cd gcst
+python3 ../manage.py test gcst
+

gcst/admin.py

@@ -0,0 +1,17 @@
+from django.contrib import admin
+from .models import Fcst,Loc
+
+admin.site.register(Fcst)
+admin.site.register(Loc)
+# OR
+#class PollAdmin(admin.ModelAdmin):
+#	fields = ['pub_date', 'question']
+#admin.site.register(Poll, PollAdmin)
+# OR
+#class PollAdmin(admin.ModelAdmin):
+#fieldsets = [
+#	(None,               {'fields': ['question']}),  # None=no title for that fieldset
+#	('Date information', {'fields': ['pub_date']}),
+#		OR ('Date information', {'fields': ['pub_date'], 'classes': ['collapse']}),
+#	]
+

gcst/appinfo.py

@@ -0,0 +1,7 @@
+
+appname='gcst'
+
+def makepath(path='.'):
+	from os.path import dirname,join as pathjoin
+	return pathjoin(dirname(__file__),path)
+

gcst/locate.py

@@ -0,0 +1,55 @@
+
+from gcst.appinfo import appname,makepath
+appdir=makepath()
+
+zipcodelines=open('%s/zipcode.csv'%(appdir)).read().split('\n')
+zip2latlon=dict([(zipc,(lat,lon,city,state))
+    for zipc,city,state,lat,lon,tz,dst in
+    [line.replace('"','').split(',') for line in zipcodelines if line.strip()]])
+city2zip=dict([((city.lower()+','+state.lower()),zipc)
+    for zipc,city,state,lat,lon,tz,dst in
+    [line.replace('"','').split(',') for line in zipcodelines if line.strip()]])
+del zipcodelines
+#TODO move dicts into db
+#TODO key by state,city; fuzzy text search?; convert full state name eg 'new jersey'->'nj'
+
+def find(loc):
+    '''
+        loc can be either zipcode or city,state
+        >>> locationInfo = find('08540')   # returns {'city': 'Princeton', 'lat': '40.357439', 'loc': '08540', 'lon': '-74.64922', 'state': 'NJ', 'zipc': '08540'}
+        >>> tuple(locationInfo[key] for key in 'city state lat lon zipc'.split())
+        ('Princeton', 'NJ', '40.357439', '-74.64922', '08540')
+        >>> locationInfo = find('Princeton, NJ')
+        >>> tuple(locationInfo[key] for key in 'city state lat lon zipc'.split())
+        ('Princeton', 'NJ', '40.349206', '-74.652811', '08544')
+        >>> locationInfo = find('princeton nj')   # returns {'city': 'Princeton', 'l': 'princeton', 'lat': '40.349206', 'loc': 'princeton nj', 'lon': '-74.652811', 'state': 'NJ', 'zipc': '08544'}
+        >>> tuple(locationInfo[key] for key in 'city state lat lon zipc'.split())
+        ('Princeton', 'NJ', '40.349206', '-74.652811', '08544')
+        '''
+    seekloc=loc.strip()
+    zipc=None
+    import re
+    if re.match(r'\d{5}.*',seekloc):
+        # loc is a zipcode
+        info=zip2latlon.get(seekloc[:5])
+        if info:
+            lat,lon,city,state=info
+            zipc=loc
+    else:
+        # loc is a city,state
+        if ',' not in seekloc and ' ' in seekloc:
+            # replace the last space with comma
+            l,r=seekloc.rsplit(' ',1)
+            seekloc=l+', '+r
+            del l,r
+        seekloc=seekloc.replace(' ','').lower()
+        zipc=city2zip.get(seekloc)
+        if zipc:
+            lat,lon,city,state=zip2latlon[zipc]
+    if zipc:
+        data=dict((k,v) for k,v in locals().items()
+            if k in 'loc zipc lat lon city state')
+    else:
+        data={'errmsg':'cannot find "%s", please try again'%(loc)}
+    return data
+

gcst/models.py

@@ -0,0 +1,17 @@
+from django.db import models
+
+class Loc(models.Model):
+    zipcode=models.CharField(max_length=5)
+    city=models.CharField(max_length=20)
+    state=models.CharField(max_length=2)
+    lat=models.FloatField()
+    lon=models.FloatField()
+    def __unicode__(self):
+        return 'Loc for %s'%(self.zipcode)
+
+class Fcst(models.Model):
+    ondelete = False
+    loc=models.ForeignKey(Loc, ondelete)
+    def __unicode__(self):
+        return 'Fcst for %s'%(self.loc)
+

gcst/readFcst.py

@@ -0,0 +1,119 @@
+
+from os import access,makedirs,O_RDONLY
+from collections import namedtuple, defaultdict
+from dateutil.parser import parse as parseDate
+from lxml import etree
+try:
+    from StringIO import StringIO
+except ImportError:
+    from io import StringIO, BytesIO
+try:
+    from urllib.request import urlopen
+except ImportError:
+    from urllib import urlopen
+from gcst.util import missing
+
+def getnewdata(zipc, lat, lon, cacheData):
+    '''get xml forecast from weather.gov [not from cache]'''
+    slots = dict(
+        # eg ringoes http://forecast.weather.gov/MapClick.php?lat=40.44659793594707&lon=-74.8513979210764&FcstType=digitalDWML
+        dataurl=   'http://forecast.weather.gov/MapClick.php?lat=%s&lon=%s&FcstType=digitalDWML'%(lat,lon),
+        tabularurl='http://forecast.weather.gov/MapClick.php?lat=%s&lon=%s&FcstType=digital'%(lat,lon),
+        humanurl=  'http://forecast.weather.gov/MapClick.php?lat=%s&lon=%s'%(lat,lon),
+    )
+    #print(dataurl)
+    xml=urlopen(slots['dataurl']).read()
+    if cacheData and zipc:
+        if not os.access(appcachedir,O_RDONLY):
+            makedirs(appcachedir)
+            open('%s/%s.xml'%(appcachedir,zipc),'w').write(xml)
+    #todo record age/expirationDate of this fcst
+    return xml, slots
+
+def startparse(xml):
+    '''create parse tree and parse the list of starttimes
+      [of the hourly time segments in the forecast].
+      see sample xml forecast at:
+        http://forecast.weather.gov/MapClick.php?lat=40.357439&lon=-74.64922&FcstType=digitalDWML
+    '''
+    tree   = etree.parse(BytesIO(xml))
+    starttimes=[parseDate(starttime) for starttime in tree.xpath('data/time-layout/start-valid-time/text()')]
+    return tree,starttimes
+
+def getdata(location, cacheData):
+    '''get data [from cache or fresh] and start parse'''
+    zipc,lat,lon=(location[k] for k in 'zipc,lat,lon'.split(','))
+    xml = None
+    if cacheData:
+        try:
+            xml = open('%s/%s.xml'%(appcachedir,zipc)).read()
+            tree, starttimes = startparse(xml)
+            fcststart = starttimes[0]
+            if fcststart - dt.now()<0.1*hour:
+                xml = None
+        except Exception as e:
+            pass
+    if not xml:
+        xml, slots = getnewdata(zipc, lat, lon, cacheData)
+        tree, starttimes = startparse(xml)
+    return tree, starttimes, slots
+
+def getFcstData(location, cacheData):
+    # parameters section contains temperature [of several types], cloud-amount, wind-speed, etc
+    tree, starttimes, slots = getdata(location, cacheData)
+    #alt=tree.xpath('data/location/height/text()')[0]
+    #loc=tree.xpath('data/location/city/text()')[0]
+    def getFcstDt(tree, slots):
+        '''get date and time of the forecast as strings'''
+        fcstDt=tree.xpath('head/product/creation-date/text()')[0]
+        date,time=fcstDt.split('T')
+        time,tz=time.rsplit('-',1)
+        time=time.rsplit(':',1)[0]
+        slots.update(dict(
+            fcstAsOfDate = date,
+            fcstAsOfTime = time,
+        ))
+    def getMoreWthrInfoUrl(tree, slots):
+        '''get url for "more weather info"'''
+        url=tree.xpath('data/moreWeatherInformation/text()')[0]
+        slots.update(dict(
+            moreWthrInfo = url
+        ))
+    getMoreWthrInfoUrl(tree, slots)
+    getFcstDt(tree, slots)
+    els=[el for el in tree.xpath('data/parameters/node()') if type(el)==etree._Element]
+    def conv(typ,val):
+        # todo should use typ eg for weather-conditions as well
+        return 0 if val is missing else int(val) if typ!='floating' else float(val)
+    def dataname(el):
+        return '-'.join([x
+            for x in (el.attrib.get('type','').replace(' ',''),el.tag.replace('-',''))
+                if x])
+    data=dict([
+        (dataname(el),[
+            conv(
+                el.attrib.get('type'),
+                val.text if val.text is not None else missing )
+            for val in el if el.tag!='weather'])
+        for el in els])
+    WeatherInfo=namedtuple('WeatherInfo','types probs prob')
+    def weatherSegment(elattribs):
+        weather=defaultdict(list)
+        for elattrib in elattribs:
+            for xmlattrib,mykey in (('weather-type','types'),('coverage','probability')):
+                if xmlattrib in elattrib:
+                    weather[mykey].append(elattrib[xmlattrib])
+        # todo check that all probability's are equal
+        # todo is first type listed somehow 'primary'
+        # we use [0:1] for 1st element because slices can return None w/o error
+        return WeatherInfo(
+            types=weather['types'],
+            probs=weather['probability'],
+            prob=weather['probability'][0] if len(weather['probability']) else None)
+    data['weather']=[
+        weatherSegment(wconds.attrib for wconds in el.getchildren())
+            for el in tree.xpath('data/parameters/weather/weather-conditions')
+                if type(el)==etree._Element]
+    #print(data.keys())
+    return data, starttimes, slots
+