Merge pull request #38 from Semalab/release_12_3

Release_12_3 -> main

Makefile

@@ -31,6 +31,7 @@ out/ai_engine.tar.gz: $(AI_ENGINE_PATH)
 ai-engine-models: $(AI_ENGINE_PATH)/.env.production
 	$(eval include $(AI_ENGINE_PATH)/.env.production)
 	aws s3 sync "s3://sagemaker-ai-code-monitor-experiments/${TUNED_MODEL}" "out/${TUNED_MODEL}"
+	# TODO: cleanup any older models
 
 $(eval $(call build-jar,$(BACKEND_CORE_PATH),))
 $(eval $(call build-jar,$(BACKEND_ACTIVITYPERSISTENCE_PATH),out/backend-core/backend-core.jar))

agent/strategy/ai_engine/__init__.py

@@ -1,16 +1,45 @@
+import logging
+import multiprocessing
+
 from agent.utils import run_logged
 
+GB_PER_THREAD = 2
+
+
 class GBOM:
     """
     runs AI Code monitor on given directories
     """
 
+    def freemem(self):
+        """
+        Returns the free memory in GB.
+        Note that this is Linux-specific, which is fine because we are always
+        running in a Docker container.
+        """
+        with open("/proc/meminfo") as file:
+            for line in file:
+                if "MemAvailable" in line:
+                    kbfree = line.split()[1]
+                    return float(kbfree) / (1024 * 1024)
+
     def run(self, directories):
+        # We can use as many threads as available CPUs, but we need to cap the
+        # number so that each thread has GB_PER_THREAD of memory available.
+        gbfree = self.freemem() or GB_PER_THREAD
+        threads = min(multiprocessing.cpu_count(), max(1, int(gbfree / GB_PER_THREAD)))
+
+        logger = logging.getLogger(__name__)
+        logger.info(f"Running GBOM with {threads} threads (free memory: {gbfree:.2f} GB)")
+
         run_logged(
             [
                 "ai_engine",
                 directories.repository,
-                "--output-dir", directories.mkdir("GBOM"),
+                "--output-dir",
+                directories.mkdir("GBOM"),
+                "--cpu_count",
+                str(threads),
             ],
-            log_dir=directories.log_dir
+            log_dir=directories.log_dir,
         )

agent/strategy/oss/dependencies/matcher.py

@@ -5,14 +5,26 @@
 # constructed by looking at `rg globsForDefinitionFiles` from within the ort repo,
 # https://github.com/oss-review-toolkit/ort
 PATTERNS = {
+    "Gulp (not ORT)": ["gulpfile.js"],
+    "ant (not ORT)": ["build.xml"],
+    "Rake (not ORT)": ["Rakefile"],
+    "setuptools (not ORT)": ["setup.cfg"],
     "stack": ["stack.yaml"],
-    "npm": ["package.json", "pnpm-lock.yaml"],
+    "npm": ["package.json", "pnpm-lock.yaml","package-lock.json"],
     "spdxdocumentfile": ["*.spdx.yml", "*.spdx.yaml", "*.spdx.json"],
     "cocoapods": ["Podfile"],
     "bundler": ["Gemfile", "Gemfile.lock"],
     "conan": ["conanfile*.txt", "conanfile*.py"],
     "carthage": ["Cartfile.resolved"],
-    "nuget": ["*.csproj", "*.fsproj", "*.vcxproj", "packages.config", "project.lock.json", "project.assets.json", "packages.lock.json"],
+    "nuget": [
+        "*.csproj",
+        "*.fsproj",
+        "*.vcxproj",
+        "packages.config",
+        "project.lock.json",
+        "project.assets.json",
+        "packages.lock.json",
+    ],
     "composer": ["composer.json"],
     "cargo": ["Cargo.toml", "Cargo.lock"],
     "pip": ["*requirements*.txt", "setup.py"],

docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM buildpack-deps:bookworm as build-cppcheck
+FROM buildpack-deps:bookworm AS build-cppcheck
 
 RUN set -eux; \
     apt-get update; \
@@ -28,7 +28,7 @@ RUN set -eux; \
 
 
 # Build OpenSSL 1.1 for .NET Core 3.1 -- remove when upgrading .NET
-FROM buildpack-deps:bookworm as build-openssl
+FROM buildpack-deps:bookworm AS build-openssl
 
 WORKDIR /root/openssl
 RUN set -eux; \
@@ -68,7 +68,7 @@ RUN set -eux; \
 
 RUN echo 'gem: --no-document' >> ~/.gemrc
 
-RUN python -m pip install --no-cache-dir poetry==1.7.1
+RUN python -m pip install --no-cache-dir poetry==1.8.3
 
 # Node.js LTS
 ARG NODE_VERSION=20.12.0
@@ -208,15 +208,15 @@ RUN python -m pip install --no-cache-dir \
 # rebuild all other dependencies when updating the database.
 
 # dependency_check: https://github.com/jeremylong/DependencyCheck
-ARG DEPENDENCY_CHECK_VERSION=9.0.7
+ARG DEPENDENCY_CHECK_VERSION=10.0.4
 RUN --mount=type=bind,source=cache,target=cache \
     set -eux; \
     curl -fsSL \
         https://github.com/jeremylong/DependencyCheck/releases/download/v${DEPENDENCY_CHECK_VERSION}/dependency-check-${DEPENDENCY_CHECK_VERSION}-release.zip \
         --output dependency-check.zip \
     ; \
     unzip dependency-check.zip; \
-    printf '%s\n%s\n%s' \
+    printf '%s\n' \
         '#!/bin/bash' \
         'cd /dependencies/dependency-check/bin/' \
         './dependency-check.sh "$@"' \
@@ -232,15 +232,14 @@ ADD out/ai_engine.tar.gz /dependencies/ai_engine/
 COPY out/tuned_models/ /dependencies/ai_engine/tuned_models/
 WORKDIR /dependencies/ai_engine
 # hadolint ignore=DL3042
-RUN --mount=type=cache,target=/root/.cache/pip set -eux; \
-    python -m venv .venv; \
-    source .venv/bin/activate; \
-    pip install -r requirements.txt -r code_chunker/requirements.txt; \
+RUN set -eux; \
+    poetry install --no-interaction --no-ansi; \
     cp .env.production .env; \
-    printf '%s\n%s\n%s' \
+    printf '%s\n' \
         '#!/bin/bash' \
         'cd /dependencies/ai_engine' \
-        '.venv/bin/python main.py "$@"' \
+        'unset VIRTUAL_ENV' \
+        'poetry run python main.py "$@"' \
         > /usr/local/bin/ai_engine \
     ; \
     chmod +x /usr/local/bin/ai_engine
@@ -270,6 +269,6 @@ COPY agent/ agent/
 RUN poetry install
 
 RUN --mount=type=bind,source=.git,target=.git \
-    git describe > version.txt
+    git describe --tags > version.txt
 
 ENTRYPOINT ["poetry", "run", "cli"]

main.py

@@ -60,7 +60,6 @@ def main(repository: Path, output: Path, scantypes: tuple[str, ...]):
             BackendAnalysis("backend-gitblame"),
             GBOM()
         ]
-
         directories = Directories(repository=repository.path, output=archive_root)
 
         logging.basicConfig(

scripts/agent

@@ -12,7 +12,7 @@ print-usage() {
   if [ "$#" != 0 ]; then
     printf '%s\n\n' "$@" >&2
   fi
-  printf '%s\n' 'Usage: agent.sh <repository> <output-directory>' >&2
+  printf '%s\n' 'Usage: agent <repository> <output-directory>' >&2
   printf '\n' >&2
   printf '%s\n' 'Runs a Sema scan on the provided <repository>, and outputs a .zip file in ' >&2
   printf '%s\n' '<output-directory>. This .zip file must be sent to Sema for further analysis.' >&2