Merge pull request #94 from cyroxx/fix-test-changes-workflow

Fix test changes workflow

.github/workflows/test_scraper_changes.yml

@@ -8,6 +8,8 @@ jobs:
     name: Show changes in scraper results
     steps:
       - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
       - name: Setup python
         uses: actions/setup-python@v1
         with:

jedeschule/pipelines/db_pipeline.py

@@ -11,9 +11,14 @@
 from jedeschule.pipelines.school_pipeline import SchoolPipelineItem
 
 Base = declarative_base()
-engine = create_engine(os.environ.get("DATABASE_URL"), echo=False)
-Session = sessionmaker(bind=engine)
-session = Session()
+
+
+def get_session():
+    engine = create_engine(os.environ.get("DATABASE_URL"), echo=False)
+    Session = sessionmaker(bind=engine)
+    session = Session()
+
+    return session
 
 
 class School(Base):
@@ -36,6 +41,8 @@ class School(Base):
 
     @staticmethod
     def update_or_create(item: SchoolPipelineItem) -> School:
+        session = get_session()
+
         school = session.query(School).get(item.info['id'])
         if school:
             session.query(School).filter_by(id=item.info['id']).update({**item.info, 'raw': item.item})
@@ -48,6 +55,7 @@ class DatabasePipeline(object):
     def process_item(self, item, spider):
         school = School.update_or_create(item)
         try:
+            session = get_session()
             session.add(school)
             session.commit()
         except SQLAlchemyError as e:

requirements.txt

@@ -1,5 +1,5 @@
 alembic==1.3.3
-Scrapy==1.7.3
+Scrapy==2.4.1
 requests==2.20.0
 wget==3.2
 xlrd==1.1.0

test_changes.py

@@ -24,8 +24,6 @@ def get_clean_item(data):
 
 
 def compare_schools(new_school, old_school):
-    print()
-    print(f"Comparing {new_school.get('id')}")
     new_school = sort_dict(new_school)
     old_school = sort_dict(old_school)
 
@@ -46,12 +44,19 @@ def main():
     data = load_data()
     for school in data[:10]:
         school_id = school.get('info').get('id')
+
+        print()
+        print('#'*10, f'Comparing {school_id}')
+
+        upstream_data = {}
         try:
             upstream_data = fetch_data(school_id)
             upstream_data.pop('raw')
-            compare_schools(school.get('info'), upstream_data)
-        except HTTPError:
-            print(f"Could not fetch old data for school-id {school_id}")
+        except HTTPError as e:
+            print(f"WARN: Could not fetch old data for school-id {school_id}: {e}")
+            print()
+
+        compare_schools(school.get('info'), upstream_data)
 
 
 if __name__ == "__main__":

test_changes.sh

@@ -2,7 +2,16 @@
 
 set -e
 
-CHANGED_SCRAPERS=$(git whatchanged --name-only --pretty="" origin..HEAD  |
+if [ $CI ]
+then
+  HEAD_REF=${GITHUB_REF}
+else
+  HEAD_REF="HEAD"
+fi
+
+echo "Using head reference: ${HEAD_REF}"
+
+CHANGED_SCRAPERS=$(git whatchanged --name-only --pretty="" origin/master..${HEAD_REF}  |
                   grep spiders |
                   grep -v helper |
                   sed 's/jedeschule\/spiders\///' |

test_models.py

@@ -7,7 +7,7 @@
 
 from jedeschule.items import School
 from jedeschule.pipelines.school_pipeline import SchoolPipelineItem
-from jedeschule.pipelines.db_pipeline import School as DBSchool, session
+from jedeschule.pipelines.db_pipeline import School as DBSchool, get_session
 
 
 class TestSchoolItem(Item):
@@ -23,6 +23,7 @@ def test_import_new(self):
         item = dict(name='Test Schule', nr=1)
         school_item: SchoolPipelineItem = SchoolPipelineItem(info=info, item=item)
         db_item = DBSchool.update_or_create(school_item)
+        session = get_session()
         session.add(db_item)
         session.commit()
 
@@ -40,6 +41,7 @@ def test_import_existing(self):
         item = dict(name='Test Schule', nr=1)
         school_item: SchoolPipelineItem = SchoolPipelineItem(info=info, item=item)
         db_item = DBSchool.update_or_create(school_item)
+        session = get_session()
         session.add(db_item)
         session.commit()