From 1f0d801b0cf7eceb8020f7616f9dc2c6c4e7b93a Mon Sep 17 00:00:00 2001 From: Dave Brondsema Date: Sun, 5 Jan 2025 12:34:04 -0500 Subject: [PATCH 1/8] run 2to3 converter --- gridsplitter/slice.py | 6 +-- waznexserver/config.py | 4 +- waznexserver/init_data.py | 14 +++--- waznexserver/models.py | 4 +- waznexserver/process_grid.py | 44 +++++++++---------- waznexserver/tests/hammeruploads/hammerit.py | 8 ++-- .../tests/hammeruploads/poster/encode.py | 22 +++++----- .../hammeruploads/poster/streaminghttp.py | 42 +++++++++--------- waznexserver/waznexserver.py | 4 +- 9 files changed, 74 insertions(+), 74 deletions(-) diff --git a/gridsplitter/slice.py b/gridsplitter/slice.py index c9b287f..cb45d81 100644 --- a/gridsplitter/slice.py +++ b/gridsplitter/slice.py @@ -289,7 +289,7 @@ def FindRow(dots, tlDotIndex, rowNum): rightDot = rightRowDots[row + 1] rowVec = (dots[rightDot][0] - dots[leftDot][0], dots[rightDot][1] - dots[leftDot][1]) except: - print("rightRowDots: {}".format(rightRowDots)) + print(("rightRowDots: {}".format(rightRowDots))) for col, topDot in enumerate(topColumnDots[1:-1]): # Generate a column-wide vector @@ -329,14 +329,14 @@ def FindRow(dots, tlDotIndex, rowNum): # Parse parameters if(len(sys.argv) <= 1 or sys.argv[1] == "--help"): print("Usage: slice.py inputfile [color [outputdir [width height]]]") - print + print() print("inputfile = The input image file") print("color = The dot color to search for: 0 = Red, 1 = Green, 2 = Blue") print("outputdir = The directory to output slices to. Defaults to") print(" a directory with the same name as the input file.") print("width = Desired width, in pixels, of output image.") print("height = Desired height, in pixels, of output image.") - print + print() print("Exit status for this script is the number of slices output.") exit(0) diff --git a/waznexserver/config.py b/waznexserver/config.py index 4198e3b..68a69a1 100644 --- a/waznexserver/config.py +++ b/waznexserver/config.py @@ -12,10 +12,10 @@ IMAGE_FOLDER = os.path.join(DATA_FOLDER, 'images') DOWNSIZED_FOLDER = os.path.join(DATA_FOLDER, 'downsized') THUMBNAIL_FOLDER = os.path.join(DATA_FOLDER, 'thumbnails') -ALLOWED_EXTENSIONS = set(['png', 'PNG', +ALLOWED_EXTENSIONS = {'png', 'PNG', 'jpg', 'JPG', 'jpeg', 'JPEG', - 'gif', 'GIF']) + 'gif', 'GIF'} FILE_NAME_DT_FORMAT = '%Y-%m-%dT%H%M%S' PRETTY_DT_FORMAT = '%m/%d/%Y %I:%M:%S %p' ENABLE_GRIDSPLITTER = True diff --git a/waznexserver/init_data.py b/waznexserver/init_data.py index 2ddca5a..52f8622 100755 --- a/waznexserver/init_data.py +++ b/waznexserver/init_data.py @@ -5,25 +5,25 @@ import os import sys -from waznexserver import db -import models -import config +from .waznexserver import db +from . import models +from . import config if __name__ == '__main__': # Create data dirs data_folders = [df for df in dir(config) if df.endswith('_FOLDER')] for folder in data_folders: newdir = os.path.abspath(getattr(config, folder)) - print newdir + print(newdir) if not os.path.exists(newdir): # If it doesn't exist, try to make it try: os.mkdir(newdir) except OSError: - print "Unable to create data directory: " + newdir - print "Error was: %s" % (sys.exc_info()[1],) + print("Unable to create data directory: " + newdir) + print("Error was: %s" % (sys.exc_info()[1],)) exit(1) if not os.path.exists(newdir): # Make sure it's there now - print "Unable to find or create data directory: " + newdir + print("Unable to find or create data directory: " + newdir) exit(1) # Create database diff --git a/waznexserver/models.py b/waznexserver/models.py index f6bb353..9e3d90f 100644 --- a/waznexserver/models.py +++ b/waznexserver/models.py @@ -4,8 +4,8 @@ import os from flask_sqlalchemy import SQLAlchemy -from waznexserver import app -from waznexserver import db +from .waznexserver import app +from .waznexserver import db # Image Levels (basic thumbnails, full grid) diff --git a/waznexserver/process_grid.py b/waznexserver/process_grid.py index a12e86b..c66505c 100755 --- a/waznexserver/process_grid.py +++ b/waznexserver/process_grid.py @@ -7,10 +7,10 @@ import shutil import subprocess import sys -from waznexserver import app -from waznexserver import db -import models -import config +from .waznexserver import app +from .waznexserver import db +from . import models +from . import config from PIL import Image @@ -34,9 +34,9 @@ def run_basic_transforms(grid_image): thumb.save(grid_image.get_thumbnail_path(), "JPEG") except: - print "Error while performing basic transforms on %s" % ( - grid_image.filename,) - print "Error was: %s" % (sys.exc_info()[1],) + print("Error while performing basic transforms on %s" % ( + grid_image.filename,)) + print("Error was: %s" % (sys.exc_info()[1],)) return False return True @@ -45,21 +45,21 @@ def run_basic_transforms(grid_image): def run_gridsplitter(grid_image): # Check that configuration variables exist if not config.GRIDSPLITTER_PYTHON or not config.GRIDSPLITTER_SLICER: - print "GridSplitter is not configured. Check your config.py." + print("GridSplitter is not configured. Check your config.py.") return False # Check validity of GRIDSPLITTER_PYTHON slicer_python = os.path.abspath(config.GRIDSPLITTER_PYTHON) if not os.path.exists(slicer_python): - print 'Aborting: Could not find GridSplitter Python.' - print 'Tried: %s' % (slicer_python,) + print('Aborting: Could not find GridSplitter Python.') + print('Tried: %s' % (slicer_python,)) return False # Check validity of GRIDSPLITTER_SLICER slicer = os.path.abspath(config.GRIDSPLITTER_SLICER) if not os.path.exists(slicer): - print 'Aborting: Could not find GridSplitter.' - print 'Tried: %s' % (slicer,) + print('Aborting: Could not find GridSplitter.') + print('Tried: %s' % (slicer,)) return False # Run the splitter for this image @@ -104,9 +104,9 @@ def verify_gridsplitter(grid_image): max_ct = config.GRIDSPLITTER_MAX_COLS *config.GRIDSPLITTER_MAX_ROWS cell_ct = len(cells) if (cell_ct < min_ct) or (cell_ct > max_ct): - print("Cell count was out of range %d-%d:%d") % (min_ct, + print(("Cell count was out of range %d-%d:%d") % (min_ct, max_ct, - cell_ct) + cell_ct)) return False # Verify each cell is within MIN and MAX rows and cols high_col = -1 @@ -117,7 +117,7 @@ def verify_gridsplitter(grid_image): row = int(parts[2]) if col < 0 or col > config.GRIDSPLITTER_MAX_COLS or\ row < 0 or row > config.GRIDSPLITTER_MAX_ROWS: - print "Column or row count was incorrect" + print("Column or row count was incorrect") return False if col > high_col: high_col = col @@ -125,7 +125,7 @@ def verify_gridsplitter(grid_image): high_row = row if (high_col < (config.GRIDSPLITTER_MIN_COLS - 1)) or\ (high_row < (config.GRIDSPLITTER_MIN_ROWS - 1)): - print "Too few rows or columns found." + print("Too few rows or columns found.") return False # TODO Verify images form an actual grid @@ -148,28 +148,28 @@ def verify_gridsplitter(grid_image): basic_result = run_basic_transforms(g) if basic_result: g.level = models.IMAGELEVEL_BASIC - print "Basic OK" + print("Basic OK") else: g.status = models.IMAGESTATUS_BAD - print "Basic Failed" + print("Basic Failed") # Do advanced image transforms if basic_result and config.ENABLE_GRIDSPLITTER: gs_result = run_gridsplitter(g) if gs_result: g.level = models.IMAGELEVEL_GRID - print "GridSplitter OK" + print("GridSplitter OK") else: # Uncomment to mark it bad #g.status = models.IMAGESTATUS_BAD - print "GridSplitter Failed" + print("GridSplitter Failed") if basic_result: g.status = models.IMAGESTATUS_DONE except: - print "Unknown error while processing image: %s" % (g.filename,) - print "Error was: %s" % (sys.exc_info()[1],) + print("Unknown error while processing image: %s" % (g.filename,)) + print("Error was: %s" % (sys.exc_info()[1],)) g.status = models.IMAGESTATUS_BAD finally: db.session.commit() diff --git a/waznexserver/tests/hammeruploads/hammerit.py b/waznexserver/tests/hammeruploads/hammerit.py index 554d68a..b76ecbe 100644 --- a/waznexserver/tests/hammeruploads/hammerit.py +++ b/waznexserver/tests/hammeruploads/hammerit.py @@ -6,7 +6,7 @@ import os from poster.encode import multipart_encode from poster.streaminghttp import register_openers -import urllib2 +import urllib.request, urllib.error, urllib.parse UPLOADS_PATH = "/home/ben/Projects/WaznexServer/waznexserver/tests/hammeruploads/uploads" UPLOAD_URL = "http://waznex-dev.clusterbleep.net/upload/" @@ -16,11 +16,11 @@ def upload_images(): image_list = os.listdir(UPLOADS_PATH) for image in image_list: try: - print "Uploading: " + image + print("Uploading: " + image) register_openers() datagen, headers = multipart_encode({"file": open(UPLOADS_PATH + '/' + image, "rb")}) - request = urllib2.Request(UPLOAD_URL, datagen, headers) - print urllib2.urlopen(request).read() + request = urllib.request.Request(UPLOAD_URL, datagen, headers) + print(urllib.request.urlopen(request).read()) except: pass # Skip files that don't work diff --git a/waznexserver/tests/hammeruploads/poster/encode.py b/waznexserver/tests/hammeruploads/poster/encode.py index cf2298d..d85451c 100644 --- a/waznexserver/tests/hammeruploads/poster/encode.py +++ b/waznexserver/tests/hammeruploads/poster/encode.py @@ -21,7 +21,7 @@ def gen_boundary(): bits = random.getrandbits(160) return sha.new(str(bits)).hexdigest() -import urllib, re, os, mimetypes +import urllib.request, urllib.parse, urllib.error, re, os, mimetypes try: from email.header import Header except ImportError: @@ -34,16 +34,16 @@ def encode_and_quote(data): if data is None: return None - if isinstance(data, unicode): + if isinstance(data, str): data = data.encode("utf-8") - return urllib.quote_plus(data) + return urllib.parse.quote_plus(data) def _strify(s): """If s is a unicode string, encode it to UTF-8 and return the results, otherwise return str(s), or None if s is None""" if s is None: return None - if isinstance(s, unicode): + if isinstance(s, str): return s.encode("utf-8") return str(s) @@ -86,7 +86,7 @@ def __init__(self, name, value=None, filename=None, filetype=None, if filename is None: self.filename = None else: - if isinstance(filename, unicode): + if isinstance(filename, str): # Encode with XML entities self.filename = filename.encode("ascii", "xmlcharrefreplace") else: @@ -153,7 +153,7 @@ def from_params(cls, params): MultipartParam object names must match the given names in the name,value pairs or mapping, if applicable.""" if hasattr(params, 'items'): - params = params.items() + params = list(params.items()) retval = [] for item in params: @@ -306,7 +306,7 @@ def get_headers(params, boundary): """Returns a dictionary with Content-Type and Content-Length headers for the multipart/form-data encoding of ``params``.""" headers = {} - boundary = urllib.quote_plus(boundary) + boundary = urllib.parse.quote_plus(boundary) headers['Content-Type'] = "multipart/form-data; boundary=%s" % boundary headers['Content-Length'] = str(get_body_size(params, boundary)) return headers @@ -326,12 +326,12 @@ def __init__(self, params, boundary, cb): def __iter__(self): return self - def next(self): + def __next__(self): """generator function to yield multipart/form-data representation of parameters""" if self.param_iter is not None: try: - block = self.param_iter.next() + block = next(self.param_iter) self.current += len(block) if self.cb: self.cb(self.p, self.current, self.total) @@ -355,7 +355,7 @@ def next(self): self.p = self.params[self.i] self.param_iter = self.p.iter_encode(self.boundary) self.i += 1 - return self.next() + return next(self) def reset(self): self.i = 0 @@ -406,7 +406,7 @@ def multipart_encode(params, boundary=None, cb=None): if boundary is None: boundary = gen_boundary() else: - boundary = urllib.quote_plus(boundary) + boundary = urllib.parse.quote_plus(boundary) headers = get_headers(params, boundary) params = MultipartParam.from_params(params) diff --git a/waznexserver/tests/hammeruploads/poster/streaminghttp.py b/waznexserver/tests/hammeruploads/poster/streaminghttp.py index 472dfd5..fc78a79 100644 --- a/waznexserver/tests/hammeruploads/poster/streaminghttp.py +++ b/waznexserver/tests/hammeruploads/poster/streaminghttp.py @@ -26,8 +26,8 @@ ... {'Content-Length': str(len(s))}) """ -import httplib, urllib2, socket -from httplib import NotConnected +import http.client, urllib.request, urllib.error, urllib.parse, socket +from http.client import NotConnected __all__ = ['StreamingHTTPConnection', 'StreamingHTTPRedirectHandler', 'StreamingHTTPHandler', 'register_openers'] @@ -58,14 +58,14 @@ def send(self, value): # NOTE: we DO propagate the error, though, because we cannot simply # ignore the error... the caller will know if they can retry. if self.debuglevel > 0: - print "send:", repr(value) + print("send:", repr(value)) try: blocksize = 8192 if hasattr(value, 'read') : if hasattr(value, 'seek'): value.seek(0) if self.debuglevel > 0: - print "sendIng a read()able" + print("sendIng a read()able") data = value.read(blocksize) while data: self.sock.sendall(data) @@ -74,21 +74,21 @@ def send(self, value): if hasattr(value, 'reset'): value.reset() if self.debuglevel > 0: - print "sendIng an iterable" + print("sendIng an iterable") for data in value: self.sock.sendall(data) else: self.sock.sendall(value) - except socket.error, v: + except socket.error as v: if v[0] == 32: # Broken pipe self.close() raise -class StreamingHTTPConnection(_StreamingHTTPMixin, httplib.HTTPConnection): +class StreamingHTTPConnection(_StreamingHTTPMixin, http.client.HTTPConnection): """Subclass of `httplib.HTTPConnection` that overrides the `send()` method to support iterable body objects""" -class StreamingHTTPRedirectHandler(urllib2.HTTPRedirectHandler): +class StreamingHTTPRedirectHandler(urllib.request.HTTPRedirectHandler): """Subclass of `urllib2.HTTPRedirectHandler` that overrides the `redirect_request` method to properly handle redirected POST requests @@ -97,7 +97,7 @@ class StreamingHTTPRedirectHandler(urllib2.HTTPRedirectHandler): the new resource, but the body of the original request is not preserved. """ - handler_order = urllib2.HTTPRedirectHandler.handler_order - 1 + handler_order = urllib.request.HTTPRedirectHandler.handler_order - 1 # From python2.6 urllib2's HTTPRedirectHandler def redirect_request(self, req, fp, code, msg, headers, newurl): @@ -120,22 +120,22 @@ def redirect_request(self, req, fp, code, msg, headers, newurl): # do the same. # be conciliant with URIs containing a space newurl = newurl.replace(' ', '%20') - newheaders = dict((k, v) for k, v in req.headers.items() + newheaders = dict((k, v) for k, v in list(req.headers.items()) if k.lower() not in ( "content-length", "content-type") ) - return urllib2.Request(newurl, + return urllib.request.Request(newurl, headers=newheaders, origin_req_host=req.get_origin_req_host(), unverifiable=True) else: - raise urllib2.HTTPError(req.get_full_url(), code, msg, headers, fp) + raise urllib.error.HTTPError(req.get_full_url(), code, msg, headers, fp) -class StreamingHTTPHandler(urllib2.HTTPHandler): +class StreamingHTTPHandler(urllib.request.HTTPHandler): """Subclass of `urllib2.HTTPHandler` that uses StreamingHTTPConnection as its http connection class.""" - handler_order = urllib2.HTTPHandler.handler_order - 1 + handler_order = urllib.request.HTTPHandler.handler_order - 1 def http_open(self, req): """Open a StreamingHTTPConnection for the given request""" @@ -152,19 +152,19 @@ def http_request(self, req): if not req.has_header('Content-length'): raise ValueError( "No Content-Length specified for iterable body") - return urllib2.HTTPHandler.do_request_(self, req) + return urllib.request.HTTPHandler.do_request_(self, req) if hasattr(httplib, 'HTTPS'): class StreamingHTTPSConnection(_StreamingHTTPMixin, - httplib.HTTPSConnection): + http.client.HTTPSConnection): """Subclass of `httplib.HTTSConnection` that overrides the `send()` method to support iterable body objects""" - class StreamingHTTPSHandler(urllib2.HTTPSHandler): + class StreamingHTTPSHandler(urllib.request.HTTPSHandler): """Subclass of `urllib2.HTTPSHandler` that uses StreamingHTTPSConnection as its http connection class.""" - handler_order = urllib2.HTTPSHandler.handler_order - 1 + handler_order = urllib.request.HTTPSHandler.handler_order - 1 def https_open(self, req): return self.do_open(StreamingHTTPSConnection, req) @@ -178,7 +178,7 @@ def https_request(self, req): if not req.has_header('Content-length'): raise ValueError( "No Content-Length specified for iterable body") - return urllib2.HTTPSHandler.do_request_(self, req) + return urllib.request.HTTPSHandler.do_request_(self, req) def get_handlers(): @@ -192,8 +192,8 @@ def register_openers(): opener object. Returns the created OpenerDirector object.""" - opener = urllib2.build_opener(*get_handlers()) + opener = urllib.request.build_opener(*get_handlers()) - urllib2.install_opener(opener) + urllib.request.install_opener(opener) return opener diff --git a/waznexserver/waznexserver.py b/waznexserver/waznexserver.py index 7a5c3d0..4f77de7 100755 --- a/waznexserver/waznexserver.py +++ b/waznexserver/waznexserver.py @@ -4,7 +4,7 @@ import os import datetime -import config +from . import config from flask import Flask from flask import flash from flask import redirect @@ -25,7 +25,7 @@ app.config.from_envvar('WAZNEXSERVER_SETTINGS', silent=True) db = SQLAlchemy(app) -import models +from . import models # https://stackoverflow.com/a/64076444/ From cb24f3879827c6c1b206099332ae68ec72f6c689 Mon Sep 17 00:00:00 2001 From: Dave Brondsema Date: Sun, 5 Jan 2025 12:36:16 -0500 Subject: [PATCH 2/8] run pyupgrade --- gridsplitter/slice.py | 2 +- waznexserver/__init__.py | 1 - waznexserver/config.py | 1 - waznexserver/init_data.py | 3 +-- waznexserver/models.py | 5 ++--- waznexserver/process_grid.py | 15 +++++++-------- waznexserver/tests/hammeruploads/hammerit.py | 1 - waznexserver/tests/hammeruploads/poster/encode.py | 6 +++--- .../tests/hammeruploads/poster/streaminghttp.py | 6 +++--- waznexserver/waznexserver.py | 1 - 10 files changed, 17 insertions(+), 24 deletions(-) diff --git a/gridsplitter/slice.py b/gridsplitter/slice.py index cb45d81..5094774 100644 --- a/gridsplitter/slice.py +++ b/gridsplitter/slice.py @@ -289,7 +289,7 @@ def FindRow(dots, tlDotIndex, rowNum): rightDot = rightRowDots[row + 1] rowVec = (dots[rightDot][0] - dots[leftDot][0], dots[rightDot][1] - dots[leftDot][1]) except: - print(("rightRowDots: {}".format(rightRowDots))) + print(f"rightRowDots: {rightRowDots}") for col, topDot in enumerate(topColumnDots[1:-1]): # Generate a column-wide vector diff --git a/waznexserver/__init__.py b/waznexserver/__init__.py index 40a96af..e69de29 100644 --- a/waznexserver/__init__.py +++ b/waznexserver/__init__.py @@ -1 +0,0 @@ -# -*- coding: utf-8 -*- diff --git a/waznexserver/config.py b/waznexserver/config.py index 68a69a1..b7ae42e 100644 --- a/waznexserver/config.py +++ b/waznexserver/config.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- import os # Configuration diff --git a/waznexserver/init_data.py b/waznexserver/init_data.py index 52f8622..5978015 100755 --- a/waznexserver/init_data.py +++ b/waznexserver/init_data.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- import os @@ -20,7 +19,7 @@ os.mkdir(newdir) except OSError: print("Unable to create data directory: " + newdir) - print("Error was: %s" % (sys.exc_info()[1],)) + print(f"Error was: {sys.exc_info()[1]}") exit(1) if not os.path.exists(newdir): # Make sure it's there now print("Unable to find or create data directory: " + newdir) diff --git a/waznexserver/models.py b/waznexserver/models.py index 9e3d90f..2a9059d 100644 --- a/waznexserver/models.py +++ b/waznexserver/models.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- import os @@ -22,7 +21,7 @@ def __init__(self, id, desc): self.desc = desc def __repr__(self): - return ''.format(self.id, self.desc) + return f'' # Image Statuses @@ -40,7 +39,7 @@ def __init__(self, id, desc): self.desc = desc def __repr__(self): - return ''.format(self.id, self.desc) + return f'' class GridItem(db.Model): diff --git a/waznexserver/process_grid.py b/waznexserver/process_grid.py index c66505c..209fb12 100755 --- a/waznexserver/process_grid.py +++ b/waznexserver/process_grid.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- @@ -34,9 +33,9 @@ def run_basic_transforms(grid_image): thumb.save(grid_image.get_thumbnail_path(), "JPEG") except: - print("Error while performing basic transforms on %s" % ( - grid_image.filename,)) - print("Error was: %s" % (sys.exc_info()[1],)) + print("Error while performing basic transforms on {}".format( + grid_image.filename)) + print(f"Error was: {sys.exc_info()[1]}") return False return True @@ -52,14 +51,14 @@ def run_gridsplitter(grid_image): slicer_python = os.path.abspath(config.GRIDSPLITTER_PYTHON) if not os.path.exists(slicer_python): print('Aborting: Could not find GridSplitter Python.') - print('Tried: %s' % (slicer_python,)) + print(f'Tried: {slicer_python}') return False # Check validity of GRIDSPLITTER_SLICER slicer = os.path.abspath(config.GRIDSPLITTER_SLICER) if not os.path.exists(slicer): print('Aborting: Could not find GridSplitter.') - print('Tried: %s' % (slicer,)) + print(f'Tried: {slicer}') return False # Run the splitter for this image @@ -168,8 +167,8 @@ def verify_gridsplitter(grid_image): g.status = models.IMAGESTATUS_DONE except: - print("Unknown error while processing image: %s" % (g.filename,)) - print("Error was: %s" % (sys.exc_info()[1],)) + print(f"Unknown error while processing image: {g.filename}") + print(f"Error was: {sys.exc_info()[1]}") g.status = models.IMAGESTATUS_BAD finally: db.session.commit() diff --git a/waznexserver/tests/hammeruploads/hammerit.py b/waznexserver/tests/hammeruploads/hammerit.py index b76ecbe..9a3ed88 100644 --- a/waznexserver/tests/hammeruploads/hammerit.py +++ b/waznexserver/tests/hammeruploads/hammerit.py @@ -26,4 +26,3 @@ def upload_images(): if __name__ == '__main__': upload_images() - \ No newline at end of file diff --git a/waznexserver/tests/hammeruploads/poster/encode.py b/waznexserver/tests/hammeruploads/poster/encode.py index d85451c..34d7507 100644 --- a/waznexserver/tests/hammeruploads/poster/encode.py +++ b/waznexserver/tests/hammeruploads/poster/encode.py @@ -47,7 +47,7 @@ def _strify(s): return s.encode("utf-8") return str(s) -class MultipartParam(object): +class MultipartParam: """Represents a single parameter in a multipart/form-data request ``name`` is the name of this parameter. @@ -186,7 +186,7 @@ def encode_hdr(self, boundary): headers = ["--%s" % boundary] if self.filename: - disposition = 'form-data; name="%s"; filename="%s"' % (self.name, + disposition = 'form-data; name="{}"; filename="{}"'.format(self.name, self.filename) else: disposition = 'form-data; name="%s"' % self.name @@ -215,7 +215,7 @@ def encode(self, boundary): if re.search("^--%s$" % re.escape(boundary), value, re.M): raise ValueError("boundary found in encoded string") - return "%s%s\r\n" % (self.encode_hdr(boundary), value) + return f"{self.encode_hdr(boundary)}{value}\r\n" def iter_encode(self, boundary, blocksize=4096): """Yields the encoding of this parameter diff --git a/waznexserver/tests/hammeruploads/poster/streaminghttp.py b/waznexserver/tests/hammeruploads/poster/streaminghttp.py index fc78a79..3d7591d 100644 --- a/waznexserver/tests/hammeruploads/poster/streaminghttp.py +++ b/waznexserver/tests/hammeruploads/poster/streaminghttp.py @@ -79,7 +79,7 @@ def send(self, value): self.sock.sendall(data) else: self.sock.sendall(value) - except socket.error as v: + except OSError as v: if v[0] == 32: # Broken pipe self.close() raise @@ -120,10 +120,10 @@ def redirect_request(self, req, fp, code, msg, headers, newurl): # do the same. # be conciliant with URIs containing a space newurl = newurl.replace(' ', '%20') - newheaders = dict((k, v) for k, v in list(req.headers.items()) + newheaders = {k: v for k, v in list(req.headers.items()) if k.lower() not in ( "content-length", "content-type") - ) + } return urllib.request.Request(newurl, headers=newheaders, origin_req_host=req.get_origin_req_host(), diff --git a/waznexserver/waznexserver.py b/waznexserver/waznexserver.py index 4f77de7..556c57a 100755 --- a/waznexserver/waznexserver.py +++ b/waznexserver/waznexserver.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- import os From 728b7db48cab2137ca6b1b16b104aaa1831366b8 Mon Sep 17 00:00:00 2001 From: Dave Brondsema Date: Sun, 5 Jan 2025 14:10:25 -0500 Subject: [PATCH 3/8] update Docker OS & python --- Dockerfile | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/Dockerfile b/Dockerfile index 95efc58..76c68be 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ -# Bionic -FROM ubuntu:18.04 +FROM ubuntu:24.04 +# 24.04 provides python 3.12 ENV DEBIAN_FRONTEND=noninteractive @@ -11,14 +11,15 @@ ENV PYTHON=$VENV/bin/python RUN mkdir $BASE -RUN apt-get update -RUN apt-get install -y nginx python-dev libjpeg62 libjpeg-dev libfreetype6 libfreetype6-dev libtiff5 libtiff5-dev libwebp6 libwebp-dev zlib1g-dev run-one -RUN apt-get install -y curl \ - && curl https://bootstrap.pypa.io/pip/2.7/get-pip.py | python \ - && pip --version -RUN pip install -U pip -RUN python -m pip install -U virtualenv -RUN virtualenv -p python2.7 $VENV +RUN apt-get update && apt-get install -y \ + nginx \ + run-one \ + python3 \ + python3-venv \ + python-is-python3 \ + && rm -rf /var/lib/apt/lists/* +RUN python -m venv $VENV +RUN $PIP install --upgrade pip WORKDIR $CODE From 0c0d77d4c053cf0b5e9c406a71bf96727d95d7cb Mon Sep 17 00:00:00 2001 From: Dave Brondsema Date: Sun, 5 Jan 2025 14:11:22 -0500 Subject: [PATCH 4/8] update deps to latest (and remove unnecessary, and convert to requirements.in/txt pattern) --- Makefile | 4 ++++ requirements.in | 8 +++++++ requirements.txt | 56 ++++++++++++++++++++++++++++++++++-------------- 3 files changed, 52 insertions(+), 16 deletions(-) create mode 100644 requirements.in diff --git a/Makefile b/Makefile index 8083e9d..1e19b8a 100644 --- a/Makefile +++ b/Makefile @@ -85,3 +85,7 @@ docker_dev: docker_dev_process: # connects to existing docker_dev and runs the processing script docker exec -it $(shell docker ps -q --filter ancestor=waznexserver) waznexserver/process_grid.py + +.PHONY: update_deps +update_deps: + uv pip compile requirements.in -o requirements.txt diff --git a/requirements.in b/requirements.in new file mode 100644 index 0000000..11e0e8f --- /dev/null +++ b/requirements.in @@ -0,0 +1,8 @@ +Flask +Flask-SQLAlchemy +gunicorn +Jinja2 +Pillow +SQLAlchemy +timeago +Werkzeug diff --git a/requirements.txt b/requirements.txt index 28dadab..4823c37 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,16 +1,40 @@ -certifi==2019.6.16 -chardet==3.0.4 -Click==7.0 -Flask==1.1.1 -Flask-SQLAlchemy==2.4.0 -gunicorn==19.9.0 -idna==2.8 -itsdangerous==1.1.0 -Jinja2==2.10.1 -MarkupSafe==1.1.1 -Pillow==6.1.0 -requests==2.22.0 -SQLAlchemy==1.3.6 -timeago==1.0.15 -urllib3==1.25.3 -Werkzeug==0.15.5 +# This file was autogenerated by uv via the following command: +# uv pip compile requirements.in -o requirements.txt +blinker==1.9.0 + # via flask +click==8.1.8 + # via flask +flask==3.1.0 + # via + # -r requirements.in + # flask-sqlalchemy +flask-sqlalchemy==3.1.1 + # via -r requirements.in +gunicorn==23.0.0 + # via -r requirements.in +itsdangerous==2.2.0 + # via flask +jinja2==3.1.5 + # via + # -r requirements.in + # flask +markupsafe==3.0.2 + # via + # jinja2 + # werkzeug +packaging==24.2 + # via gunicorn +pillow==11.1.0 + # via -r requirements.in +sqlalchemy==2.0.36 + # via + # -r requirements.in + # flask-sqlalchemy +timeago==1.0.16 + # via -r requirements.in +typing-extensions==4.12.2 + # via sqlalchemy +werkzeug==3.1.3 + # via + # -r requirements.in + # flask From 6944d135266f0da18026c6aef670a418fabb53f0 Mon Sep 17 00:00:00 2001 From: Dave Brondsema Date: Sun, 5 Jan 2025 16:48:32 -0500 Subject: [PATCH 5/8] update flask/sqlalchemy setup; use create_app factory and blueprint --- waznexserver/init_data.py | 28 ++++++++++----- waznexserver/models.py | 11 ++++-- waznexserver/process_grid.py | 24 ++++++++----- waznexserver/templates/colview.html | 2 +- waznexserver/templates/index.html | 32 ++++++++--------- waznexserver/waznexserver.py | 55 +++++++++++++++++------------ 6 files changed, 91 insertions(+), 61 deletions(-) diff --git a/waznexserver/init_data.py b/waznexserver/init_data.py index 5978015..abbb132 100755 --- a/waznexserver/init_data.py +++ b/waznexserver/init_data.py @@ -1,15 +1,15 @@ #!/usr/bin/env python - import os import sys -from .waznexserver import db -from . import models -from . import config +import config +import models +from models import db +from waznexserver import create_app -if __name__ == '__main__': - # Create data dirs + +def create_data_dirs(): data_folders = [df for df in dir(config) if df.endswith('_FOLDER')] for folder in data_folders: newdir = os.path.abspath(getattr(config, folder)) @@ -25,8 +25,10 @@ print("Unable to find or create data directory: " + newdir) exit(1) - # Create database + +def create_database(): db.create_all() + db.session.commit() # Find and add all of the ImageStatuses in models.py statuses = [s for s in dir(models) if s.startswith('IMAGESTATUS_')] @@ -34,12 +36,20 @@ id = getattr(models, status) s = models.ImageStatus(id, status.split('_',1)[1]) db.session.add(s) - + # Find and add all of the ImageLevels in models.py levels = [l for l in dir(models) if l.startswith('IMAGELEVEL_')] for level in levels: id = getattr(models, level) l = models.ImageLevel(id, level.split('_',1)[1]) db.session.add(l) - + db.session.commit() + + +if __name__ == '__main__': + create_data_dirs() + + app = create_app() + with app.app_context(): + create_database() diff --git a/waznexserver/models.py b/waznexserver/models.py index 2a9059d..6a1dac5 100644 --- a/waznexserver/models.py +++ b/waznexserver/models.py @@ -1,11 +1,16 @@ #!/usr/bin/env python - import os + +from flask import current_app as app # is this misleading? from flask_sqlalchemy import SQLAlchemy -from .waznexserver import app -from .waznexserver import db +from sqlalchemy.orm import DeclarativeBase + + +class Base(DeclarativeBase): + pass +db = SQLAlchemy(model_class=Base) # Image Levels (basic thumbnails, full grid) IMAGELEVEL_NOTHING = -1 diff --git a/waznexserver/process_grid.py b/waznexserver/process_grid.py index 209fb12..cd64780 100755 --- a/waznexserver/process_grid.py +++ b/waznexserver/process_grid.py @@ -1,16 +1,18 @@ #!/usr/bin/env python - - import os import shutil import subprocess import sys -from .waznexserver import app -from .waznexserver import db -from . import models -from . import config +import traceback + from PIL import Image +from flask import current_app as app # is this misleading? + +import config +import models +from models import db +from waznexserver import create_app def run_basic_transforms(grid_image): @@ -132,8 +134,7 @@ def verify_gridsplitter(grid_image): return True -if __name__ == '__main__': - # Find all of the images that are new +def process_new_images(): new_grids = db.session.query(models.GridItem).\ filter_by(status=models.IMAGESTATUS_NEW).\ order_by('upload_dt').all() @@ -172,4 +173,9 @@ def verify_gridsplitter(grid_image): g.status = models.IMAGESTATUS_BAD finally: db.session.commit() - + + +if __name__ == '__main__': + app = create_app() + with app.app_context(): + process_new_images() diff --git a/waznexserver/templates/colview.html b/waznexserver/templates/colview.html index 740f694..24e7d6b 100644 --- a/waznexserver/templates/colview.html +++ b/waznexserver/templates/colview.html @@ -3,7 +3,7 @@ BarCampGR Session Grid - + diff --git a/waznexserver/templates/index.html b/waznexserver/templates/index.html index 688b261..585ef34 100644 --- a/waznexserver/templates/index.html +++ b/waznexserver/templates/index.html @@ -4,23 +4,23 @@ BarCampGR Session Grid - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + - + diff --git a/waznexserver/waznexserver.py b/waznexserver/waznexserver.py index 556c57a..b18cf06 100755 --- a/waznexserver/waznexserver.py +++ b/waznexserver/waznexserver.py @@ -3,37 +3,46 @@ import os import datetime -from . import config -from flask import Flask + +from flask import Flask, Blueprint +from flask import current_app as app # is this misleading? from flask import flash from flask import redirect from flask import render_template from flask import request from flask import send_from_directory from flask import url_for -from flask_sqlalchemy import SQLAlchemy from sqlalchemy import desc from sqlalchemy import or_ import timeago -from werkzeug import secure_filename +from werkzeug.utils import secure_filename from werkzeug.security import safe_join -app = Flask(__name__) -app.config.from_object(__name__) -app.config.from_object(config) -app.config.from_envvar('WAZNEXSERVER_SETTINGS', silent=True) +import config +import models +from models import db + +main = Blueprint('main', __name__) + +def create_app(): + app = Flask(__name__) + app.config.from_object(__name__) + app.config.from_object(config) + app.config.from_envvar('WAZNEXSERVER_SETTINGS', silent=True) + + app.register_blueprint(main) -db = SQLAlchemy(app) -from . import models + models.db.init_app(app) + return app # https://stackoverflow.com/a/64076444/ -@app.template_filter('timeago') +@main.app_template_filter('timeago') def timeago_filter(date): return timeago.format(date, datetime.datetime.now()) -@app.route('/') +@main.route('/') def index(): # Fetch newest images from DB grid = db.session.query(models.GridItem).\ @@ -52,33 +61,33 @@ def index(): grid=grid, pretty_dt_format=app.config['PRETTY_DT_FORMAT']) -@app.route('/favicon.ico') +@main.route('/favicon.ico') def favicon(): filename = request.args.get('filename', 'favicon.ico') return send_from_directory(os.path.join(app.root_path, 'static', 'favicon'), filename) -@app.route('/thumbnail/') +@main.route('/thumbnail/') def show_thumbnail(filename): app.logger.info('Serving thumbnail through Flask: ' + filename) return send_from_directory(app.config['THUMBNAIL_FOLDER'], filename) -@app.route('/medium/') +@main.route('/medium/') def show_downsized(filename): app.logger.info('Serving downsized image through Flask: ' + filename) return send_from_directory(app.config['DOWNSIZED_FOLDER'], filename) -@app.route('/image/') +@main.route('/image/') def show_image(filename): app.logger.info('Serving image through Flask: ' + filename) return send_from_directory(app.config['IMAGE_FOLDER'], filename) -@app.route('/sliced//') +@main.route('/sliced//') def show_sliced(dirname, filename): app.logger.info('Serving cell image through Flask: ' + filename) return send_from_directory(app.config['SPLIT_FOLDER'], safe_join(dirname, filename)) -@app.route('/colview//') +@main.route('/colview//') def show_colview(grid_item_id, col_num): # Get column 0 - Room List # Get column col_num @@ -91,7 +100,7 @@ def show_colview(grid_item_id, col_num): cell_list=cells, ) -@app.route('/upload/', methods=['GET', 'POST']) +@main.route('/upload/', methods=['GET', 'POST']) def upload_file(): if request.method == 'POST': f = request.files['file'] @@ -114,7 +123,7 @@ def upload_file(): else: flash('Upload failed - invalid file extension.', "message-upload-fail") - return redirect(url_for('index')) + return redirect(url_for('main.index')) def allowed_file(filename): ext = False @@ -122,7 +131,7 @@ def allowed_file(filename): ext = filename.rsplit('.', 1)[1] return ext in app.config['ALLOWED_EXTENSIONS'] -@app.route('/mark_bad/', methods=['GET']) +@main.route('/mark_bad/', methods=['GET']) def mark_bad(grid_item_id): try: bgi = db.session.query(models.GridItem).\ @@ -134,8 +143,8 @@ def mark_bad(grid_item_id): except: # Invalid grid_item_id. Ignore it. pass - return redirect(url_for('index')) + return redirect(url_for('main.index')) if __name__ == '__main__': - app.run(host='0.0.0.0', port=8080) + create_app().run(host='0.0.0.0', port=8080) From 2b8af007078429239fc415f41346f1b3c28072e7 Mon Sep 17 00:00:00 2001 From: Dave Brondsema Date: Sun, 5 Jan 2025 16:49:12 -0500 Subject: [PATCH 6/8] update for latest PIL --- gridsplitter/slice.py | 2 +- waznexserver/process_grid.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/gridsplitter/slice.py b/gridsplitter/slice.py index 5094774..e24a082 100644 --- a/gridsplitter/slice.py +++ b/gridsplitter/slice.py @@ -218,7 +218,7 @@ def FindRow(dots, tlDotIndex, rowNum): # Downsize the search space to speed things up and make the search # more accurate - imageSearch = imageLuminance.resize(searchSize, Image.ANTIALIAS) + imageSearch = imageLuminance.resize(searchSize, Image.LANCZOS) # Find all of the dots in the image dots = TransformDots(FindDots(imageSearch), imageSearch, imageOriginal) diff --git a/waznexserver/process_grid.py b/waznexserver/process_grid.py index cd64780..afb4baf 100755 --- a/waznexserver/process_grid.py +++ b/waznexserver/process_grid.py @@ -23,7 +23,7 @@ def run_basic_transforms(grid_image): shutil.copy2(grid_image.get_image_path(), grid_image.get_downsized_path()) downs = Image.open(grid_image.get_downsized_path()) - downs.thumbnail((1024,1024), Image.ANTIALIAS) + downs.thumbnail((1024,1024), Image.LANCZOS) downs.save(grid_image.get_downsized_path(), "JPEG") # Copy orig and create thumbnail version @@ -31,7 +31,7 @@ def run_basic_transforms(grid_image): shutil.copy2(grid_image.get_image_path(), grid_image.get_thumbnail_path()) thumb = Image.open(grid_image.get_thumbnail_path()) - thumb.thumbnail((316,316), Image.ANTIALIAS) + thumb.thumbnail((316,316), Image.LANCZOS) thumb.save(grid_image.get_thumbnail_path(), "JPEG") except: From 010df173257d4b75e3b628b1cd48467c53df742b Mon Sep 17 00:00:00 2001 From: Dave Brondsema Date: Sun, 5 Jan 2025 16:49:33 -0500 Subject: [PATCH 7/8] better exception handling --- gridsplitter/slice.py | 2 +- waznexserver/init_data.py | 4 ++-- waznexserver/process_grid.py | 18 +++++++++--------- waznexserver/tests/hammeruploads/hammerit.py | 2 +- waznexserver/waznexserver.py | 2 +- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/gridsplitter/slice.py b/gridsplitter/slice.py index e24a082..24175d7 100644 --- a/gridsplitter/slice.py +++ b/gridsplitter/slice.py @@ -288,7 +288,7 @@ def FindRow(dots, tlDotIndex, rowNum): try:# Generate a row-wide vector rightDot = rightRowDots[row + 1] rowVec = (dots[rightDot][0] - dots[leftDot][0], dots[rightDot][1] - dots[leftDot][1]) - except: + except Exception: print(f"rightRowDots: {rightRowDots}") for col, topDot in enumerate(topColumnDots[1:-1]): diff --git a/waznexserver/init_data.py b/waznexserver/init_data.py index abbb132..b2c70c7 100755 --- a/waznexserver/init_data.py +++ b/waznexserver/init_data.py @@ -1,7 +1,7 @@ #!/usr/bin/env python import os -import sys +import traceback import config import models @@ -19,7 +19,7 @@ def create_data_dirs(): os.mkdir(newdir) except OSError: print("Unable to create data directory: " + newdir) - print(f"Error was: {sys.exc_info()[1]}") + traceback.print_exc() exit(1) if not os.path.exists(newdir): # Make sure it's there now print("Unable to find or create data directory: " + newdir) diff --git a/waznexserver/process_grid.py b/waznexserver/process_grid.py index afb4baf..824a827 100755 --- a/waznexserver/process_grid.py +++ b/waznexserver/process_grid.py @@ -20,26 +20,26 @@ def run_basic_transforms(grid_image): # Copy orig and create downsized version (1024x1024 max) app.logger.info('Generating downsized image for ' + grid_image.filename) - shutil.copy2(grid_image.get_image_path(), + shutil.copy2(grid_image.get_image_path(), grid_image.get_downsized_path()) downs = Image.open(grid_image.get_downsized_path()) downs.thumbnail((1024,1024), Image.LANCZOS) downs.save(grid_image.get_downsized_path(), "JPEG") - + # Copy orig and create thumbnail version app.logger.info('Generating thumbnail for ' + grid_image.filename) - shutil.copy2(grid_image.get_image_path(), + shutil.copy2(grid_image.get_image_path(), grid_image.get_thumbnail_path()) thumb = Image.open(grid_image.get_thumbnail_path()) thumb.thumbnail((316,316), Image.LANCZOS) thumb.save(grid_image.get_thumbnail_path(), "JPEG") - - except: + + except Exception: print("Error while performing basic transforms on {}".format( grid_image.filename)) - print(f"Error was: {sys.exc_info()[1]}") + traceback.print_exc() return False - + return True @@ -167,9 +167,9 @@ def process_new_images(): if basic_result: g.status = models.IMAGESTATUS_DONE - except: + except Exception: print(f"Unknown error while processing image: {g.filename}") - print(f"Error was: {sys.exc_info()[1]}") + traceback.print_exc() g.status = models.IMAGESTATUS_BAD finally: db.session.commit() diff --git a/waznexserver/tests/hammeruploads/hammerit.py b/waznexserver/tests/hammeruploads/hammerit.py index 9a3ed88..412902c 100644 --- a/waznexserver/tests/hammeruploads/hammerit.py +++ b/waznexserver/tests/hammeruploads/hammerit.py @@ -21,7 +21,7 @@ def upload_images(): datagen, headers = multipart_encode({"file": open(UPLOADS_PATH + '/' + image, "rb")}) request = urllib.request.Request(UPLOAD_URL, datagen, headers) print(urllib.request.urlopen(request).read()) - except: + except Exception: pass # Skip files that don't work if __name__ == '__main__': diff --git a/waznexserver/waznexserver.py b/waznexserver/waznexserver.py index b18cf06..51c70bb 100755 --- a/waznexserver/waznexserver.py +++ b/waznexserver/waznexserver.py @@ -140,7 +140,7 @@ def mark_bad(grid_item_id): bgi.status = models.IMAGESTATUS_BAD db.session.commit() flash("Marked an image as bad and removed it.", "message-removed-bad") - except: + except Exception: # Invalid grid_item_id. Ignore it. pass return redirect(url_for('main.index')) From 00f61c5eac04db785f254230e2bcc31eeff59a3b Mon Sep 17 00:00:00 2001 From: Dave Brondsema Date: Sun, 5 Jan 2025 16:55:07 -0500 Subject: [PATCH 8/8] no longer need to run on x86/64 (works on Mac Arm) --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 1e19b8a..83c86b8 100644 --- a/Makefile +++ b/Makefile @@ -70,7 +70,7 @@ run_production: sudo service waznex-process-grid restart sudo service nginx restart -DOCKER_BUILD := docker build --platform linux/amd64 -t waznexserver . +DOCKER_BUILD := docker build -t waznexserver . .PHONY: docker_build docker_build: @@ -79,7 +79,7 @@ docker_build: .PHONY: docker_dev docker_dev: # (re)builds the latest image and runs it - docker run --rm --platform linux/amd64 -p 8080:8080 -it $(shell $(DOCKER_BUILD) -q) + docker run --rm -p 8080:8080 -it $(shell $(DOCKER_BUILD) -q) .PHONY: docker_dev_process docker_dev_process: