First commit

.gitignore

@@ -0,0 +1 @@
+data/*

run_scraper.py

@@ -0,0 +1,61 @@
+# The MIT License (MIT)
+#
+# Copyright (c) 2014 Anders Bennehag
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+#
+
+import os
+import yahooscraper
+import sendemail
+import ticker_lists
+
+
+DATA_DIR = 'data'
+
+LISTS = [
+    (ticker_lists.STO_MID_CAP, 1),
+    (ticker_lists.STO_SMALL_CAP, 1),
+    (ticker_lists.WORLD_INDICES, 200),
+    (ticker_lists.OMXS30, 200),
+    ]
+
+
+def appendToFile(ticker, data):
+    ticker = ticker.replace('^', '_')
+    fname = os.path.join(DATA_DIR, ticker + '.csv')
+    with open(fname, 'wa') as f:
+        for line in data:
+            f.write(line + '\n')
+
+
+if __name__ == '__main__':
+
+    for tlist, minsamples in LISTS:
+        for ticker in tlist:
+            try:
+                #print 'Fetching {}'.format(ticker)
+                data = yahooscraper.get_intraday(ticker, minsamples)
+                appendToFile(ticker, data)
+            except Exception as e:
+                subject = 'Error parsing ticker {}'.format(ticker)
+                msg = sendemail.formatExceptionMsg(e)
+                sendemail.sendEmail('[email protected]', subject, msg)
+            else:
+                pass

sendemail.py

@@ -0,0 +1,55 @@
+# The MIT License (MIT)
+#
+# Copyright (c) 2014 Anders Bennehag
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+#
+
+import smtplib
+import string
+import traceback
+
+
+def sendEmail(to, subject, msg):
+    import google
+    frm = google.USER
+    BODY = string.join((
+        "From: %s" % frm,
+        "To: %s" % to,
+        "Subject: %s" % subject,
+        "",
+        msg
+        ), "\r\n")
+
+    server = smtplib.SMTP('smtp.gmail.com:587')
+    server.starttls()
+    server.login(google.USER, google.PASS)
+    server.sendmail(frm, to, BODY)
+    server.quit()
+
+
+def formatExceptionMsg(e):
+    return ('Caught exception: {}\n\n'.format(str(e)) +
+            'Full stacktrace:\n' + traceback.format_exc())
+
+
+def emailException(to, fname, e):
+    subject = 'ERROR: Exception in {}'.format(fname)
+    msg = formatExceptionMsg(e)
+    sendEmail(to, subject, msg)

ticker_lists.py

@@ -0,0 +1,237 @@
+
+
+WORLD_INDICES = [
+    '^OMX',
+    '^GDAXI',
+    '^FRDOW',
+    '^ESDOW',
+    '^NDX',
+    '^DJI',
+    '^SPXPM',
+    '^VIX',
+    '^N225',
+    ]
+
+OMXS30 = [
+    'ABB.ST',
+    'ALFA.ST',
+    'ASSA-B.ST',
+    'AZN.ST',
+    'ATCO-B.ST',
+    'BOL.ST',
+    'ELUX-B.ST',
+    'ERIC-B.ST',
+    'GETI-B.ST',
+    'HM-B.ST',
+    #INVE-B
+    'LUPE.ST',
+    'MTG-B.ST',
+    'NDA-SEK.ST',
+    'SAND.ST',
+    'SCA-B.ST',
+    'SEB-A.ST',
+    'SECU-B.ST',
+    'SKA-B.ST',
+    'SKF-B.ST',
+    'SSAB-A.ST',
+    'SHB-A.ST',
+    'SWED-A.ST',
+    'SWMA.ST',
+    'TEL2-B.ST',
+    'TLSN.ST',
+    'VOLV-B.ST',
+    ]
+
+
+STO_MID_CAP = [
+    'ACTI.ST',
+    'ADDT-B.ST',
+    'AERO-B.ST',
+    'AF-B.ST',
+    'ARCM.ST',
+    'AZA.ST',
+    'BACTI-B.ST',
+    'BALD-B.ST',
+    'BBTO-B.ST',
+    'BEIA-B.ST',
+    'BEIJ-B.ST',
+    'BETS-B.ST',
+    'BIOG-B.ST',
+    'BMAX.ST',
+    'BUFAB.ST',
+    'BURE.ST',
+    'CATE.ST',
+    'CCC.ST',
+    'CDON.ST',
+    'CLA-B.ST',
+    'CLAS-B.ST',
+    'COIC.ST',
+    'CORE.ST',
+    'DIOS.ST',
+    'DUNI.ST',
+    'ECEX.ST',
+    'ENRO.ST',
+    'FAG.ST',
+    'FING-B.ST',
+    'FOI-B.ST',
+    'FPAR.ST',
+    'GUNN.ST',
+    'HEBA-B.ST',
+    'HEMF.ST',
+    'HIQ.ST',
+    'HLDX.ST',
+    'IFS-B.ST',
+    'INDT.ST',
+    'ITAB-B.ST',
+    'KAHL.ST',
+    'KLED.ST',
+    'KLOV.ST',
+    'LAGR-B.ST',
+    'LIAB.ST',
+    'LOOM-B.ST',
+    'LUC.ST',
+    'MEKO.ST',
+    'MVIR-B.ST',
+    'NET-B.ST',
+    'NEWA-B.ST',
+    'NGQ.ST',
+    'NMAN.ST',
+    'NN-B.ST',
+    'NOBI.ST',
+    'NOLA-B.ST',
+    'OEM-B.ST',
+    'OPUS.ST',
+    'ORES.ST',
+    'ORX.ST',
+    'PLAZ-B.ST',
+    'PROE-B.ST',
+    'RECI-B.ST',
+    'REZT.ST',
+    'SAGA-B.ST',
+    'SAS.ST',
+    'SCST.ST',
+    'SECT-B.ST',
+    'SKIS-B.ST',
+    'SMF.ST',
+    'SNTC.ST',
+    'SWEC-B.ST',
+    'SWOL-B.ST',
+    'SYSR.ST',
+    'TETY.ST',
+    'TRI.ST',
+    'TRMO.ST',
+    'VICP-B.ST',
+    'WIHL.ST',
+    ]
+
+STO_SMALL_CAP = [
+    'ACAN-B.ST',
+    'ACAP-B.ST',
+    'ALNX.ST',
+    'ANOD-B.ST',
+    'ANOT.ST',
+    'ARISE.ST',
+    'ARP.ST',
+    'ASP.ST',
+    'AVEG-B.ST',
+    'BEGR.ST',
+    'BELE.ST',
+    'BESQ.ST',
+    'BINV.ST',
+    'BIOT.ST',
+    'BONG.ST',
+    'BORG.ST',
+    'BOUL.ST',
+    'BRG-B.ST',
+    'BTS-B.ST',
+    'CCOR-B.ST',
+    'CEVI.ST',
+    'CNTA.ST',
+    'CONS-B.ST',
+    'CSN.ST',
+    'CTT.ST',
+    'CYBE.ST',
+    'DEDI.ST',
+    'DGC.ST',
+    'DORO.ST',
+    'DURC-B.ST',
+    'ELAN-B.ST',
+    'ELEC.ST',
+    'ELOS-B.ST',
+    'ENDO.ST',
+    'ENEA.ST',
+    'EPIS-B.ST',
+    'ETX.ST',
+    'EWRK.ST',
+    'FBAB.ST',
+    'FEEL.ST',
+    'FPIP.ST',
+    'G5EN.ST',
+    'GHP.ST',
+    'GVKO-B.ST',
+    'HAV-B.ST',
+    'HEMX.ST',
+    'HMS.ST',
+    'IAR-B.ST',
+    'ICTA-B.ST',
+    'IS.ST',
+    'KABE-B.ST',
+    'KARO.ST',
+    'KDEV.ST',
+    'KNOW.ST',
+    'LAMM-B.ST',
+    'MEAB-B.ST',
+    'MIDW-B.ST',
+    'MOB.ST',
+    'MQ.ST',
+    'MSAB-B.ST',
+    'MSC-B.ST',
+    'MSON-B.ST',
+    'MULQ.ST',
+    'NAXS.ST',
+    'NETI-B.ST',
+    'NOMI.ST',
+    'NOTE.ST',
+    'NOVE.ST',
+    'NSP-B.ST',
+    'NTEK-B.ST',
+    'NVP.ST',
+    'OASM.ST',
+    'ODD.ST',
+    'OPCO.ST',
+    'ORTI-B.ST',
+    'PACT.ST',
+    'PAR.ST',
+    'PART.ST',
+    'POOL-B.ST',
+    'PREC.ST',
+    'PREV-B.ST',
+    'PRIC-B.ST',
+    'PROB.ST',
+    'PROF-B.ST',
+    'RABT-B.ST',
+    'RAY-B.ST',
+    'REJL-B.ST',
+    'RNBS.ST',
+    'RROS.ST',
+    'RSOF-B.ST',
+    'RTIM-B.ST',
+    'SEAM.ST',
+    'SEMC.ST',
+    'SENS.ST',
+    'SHEL-B.ST',
+    'SINT.ST',
+    'SOF-B.ST',
+    'SVED-B.ST',
+    'SVIK.ST',
+    'SVOL-B.ST',
+    'TAGR.ST',
+    'TRAC-B.ST',
+    'TRAD.ST',
+    'UFLX-B.ST',
+    'VBG-B.ST',
+    'VIT-B.ST',
+    'VITR.ST',
+    'VRG-B.ST',
+    'XANO-B.ST',
+]