Updates for running engine on fips-enabled hosts

Dockerfile

@@ -5,9 +5,8 @@ FROM registry.access.redhat.com/ubi8/ubi:8.4 as anchore-engine-builder
 ARG CLI_COMMIT
 
 ENV LANG=en_US.UTF-8 LC_ALL=C.UTF-8
-
 ENV GOPATH=/go
-ENV SKOPEO_VERSION=v1.2.1
+
 
 COPY . /buildsource
 WORKDIR /buildsource
@@ -19,7 +18,10 @@ RUN set -ex && \
     echo "installing OS dependencies" && \
     yum update -y && \
     yum module disable -y python36 && yum module enable -y python38 && \
-    yum install -y gcc make python38 git python38-wheel python38-devel go
+    yum install -y gcc make python38 git python38-wheel python38-devel python38-psycopg2 go  && \
+    pip3 install pip==21.0.1 && \
+    yum install -y https://download-ib01.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && \
+    yum install -y --downloadonly --downloaddir=/build_output/build_deps/ dpkg clamav clamav-update
 
 # create anchore binaries
 RUN set -ex && \
@@ -36,14 +38,6 @@ RUN set -ex && \
     echo "installing GO" && \
     mkdir -p /go
 
-RUN set -ex && \
-    echo "installing Skopeo" && \
-    git clone --branch "$SKOPEO_VERSION" https://github.com/containers/skopeo ${GOPATH}/src/github.com/containers/skopeo && \
-    cd ${GOPATH}/src/github.com/containers/skopeo && \
-    make install-binary DISABLE_CGO=1 && \
-    cp /usr/bin/skopeo /build_output/deps/ && \
-    cp default-policy.json /build_output/configs/skopeo-policy.json
-
 RUN set -ex && \
     echo "installing Syft" && \
     curl -sSfL https://raw.githubusercontent.com/anchore/syft/main/install.sh | sh -s -- -b /build_output/deps v0.19.1
@@ -52,10 +46,6 @@ RUN set -ex && \
     echo "installing Grype" && \
     curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /build_output/deps v0.13.0
 
-# stage RPM dependency binaries
-RUN yum install -y https://download-ib01.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && \
-    yum install -y --downloadonly --downloaddir=/build_output/deps/ dpkg clamav clamav-update
-
 RUN tar -z -c -v -C /build_output -f /anchore-buildblob.tgz .
 
 # Build setup section
@@ -150,7 +140,8 @@ EXPOSE ${ANCHORE_SERVICE_PORT}
 RUN set -ex && \
     yum update -y && \
     yum module disable -y python36 && yum module enable -y python38 && \
-    yum install -y python38 python38-wheel procps psmisc
+    yum install -y python38 python38-wheel procps psmisc python38-psycopg2 skopeo && \
+    pip3 install pip==21.0.1
 
 # Setup container default configs and directories
 
@@ -181,19 +172,16 @@ RUN set -ex && \
 RUN set -ex && \
     python3 -m venv /anchore-cli && \
     source /anchore-cli/bin/activate && \
-    pip3 install --no-index --find-links=./ /build_output/cli_wheels/*.whl && \
+    pip3 install --no-index --find-links=/build_output/cli_wheels/ anchorecli && \
     deactivate
 
 # Perform the anchore-engine build and install
 
 RUN set -ex && \
-    pip3 install --no-index --find-links=./ /build_output/wheels/*.whl && \
-    cp /build_output/deps/skopeo /usr/bin/skopeo && \
+    pip3 install --no-index --find-links=/build_output/wheels/ anchore-engine && \
     cp /build_output/deps/syft /usr/bin/syft && \
     cp /build_output/deps/grype /usr/bin/grype && \
-    mkdir -p /etc/containers && \
-    cp /build_output/configs/skopeo-policy.json /etc/containers/policy.json && \
-    yum install -y /build_output/deps/*.rpm && \
+    #yum install -y /build_output/deps/*.rpm && \
     rm -rf /build_output /root/.cache
 
 # Container runtime instructions

anchore_engine/analyzers/utils.py

@@ -54,7 +54,9 @@ def init_analyzer_cmdline(argv, name):
     ret["name"] = name
 
     with open(argv[0], "r") as FH:
-        ret["selfcsum"] = hashlib.md5(FH.read().encode("utf-8")).hexdigest()
+        ret["selfcsum"] = hashlib.new(
+            "md5", FH.read().encode("utf-8"), usedforsecurity=False
+        ).hexdigest()
 
     ret["imgid"] = argv[2]
 
@@ -485,11 +487,6 @@ def _get_extractable_member(
 def _checksum_member_function(tfl, member, csums=["sha256", "md5"], memberhash={}):
     ret = {}
 
-    funcmap = {
-        "sha256": hashlib.sha256,
-        "sha1": hashlib.sha1,
-        "md5": hashlib.md5,
-    }
     if member.isreg():
         extractable_member = member
     elif member.islnk():
@@ -502,7 +499,9 @@ def _checksum_member_function(tfl, member, csums=["sha256", "md5"], memberhash={
     for ctype in csums:
         if extractable_member:
             with tfl.extractfile(extractable_member) as mfd:
-                ret[ctype] = funcmap[ctype](mfd.read()).hexdigest()
+                ret[ctype] = hashlib.new(
+                    ctype, mfd.read(), usedforsecurity=False
+                ).hexdigest()
         else:
             ret[ctype] = "DIRECTORY_OR_OTHER"
 
@@ -512,12 +511,6 @@ def _checksum_member_function(tfl, member, csums=["sha256", "md5"], memberhash={
 def get_checksums_from_squashtar(squashtar, csums=["sha256", "md5"]):
     allfiles = {}
 
-    funcmap = {
-        "sha256": hashlib.sha256,
-        "sha1": hashlib.sha1,
-        "md5": hashlib.md5,
-    }
-
     try:
         results = anchore_engine.analyzers.utils.run_tarfile_member_function(
             squashtar, func=_checksum_member_function, csums=csums

anchore_engine/clients/services/catalog.py

@@ -305,10 +305,12 @@ def delete_subscription(
         self, subscription_key=None, subscription_type=None, subscription_id=None
     ):
         if subscription_key and subscription_type:
-            subscription_id = hashlib.md5(
+            subscription_id = hashlib.new(
+                "md5",
                 "+".join(
                     [self.request_namespace, subscription_key, subscription_type]
-                ).encode("utf8")
+                ).encode("utf8"),
+                usedforsecurity=False,
             ).hexdigest()
 
         return self.call_api(
@@ -325,22 +327,26 @@ def update_subscription(
         if subscription_id:
             pass
         elif subscription_key and subscription_type:
-            subscription_id = hashlib.md5(
+            subscription_id = hashlib.new(
+                "md5",
                 "+".join(
                     [self.request_namespace, subscription_key, subscription_type]
-                ).encode("utf8")
+                ).encode("utf8"),
+                usedforsecurity=False,
             ).hexdigest()
         elif subscriptiondata.get("subscription_key", None) and subscriptiondata.get(
             "subscription_type", None
         ):
-            subscription_id = hashlib.md5(
+            subscription_id = hashlib.new(
+                "md5",
                 "+".join(
                     [
                         self.request_namespace,
                         subscriptiondata.get("subscription_key"),
                         subscriptiondata.get("subscription_type"),
                     ]
-                ).encode("utf8")
+                ).encode("utf8"),
+                usedforsecurity=False,
             ).hexdigest()
         else:
             raise Exception(

anchore_engine/db/db_queue.py

@@ -54,7 +54,9 @@ def create(
 
 def generate_dataId(inobj):
     datajson = json.dumps(inobj)
-    dataId = hashlib.md5(datajson.encode("utf-8")).hexdigest()
+    dataId = hashlib.new(
+        "md5", datajson.encode("utf-8"), usedforsecurity=False
+    ).hexdigest()
     return dataId, datajson
 
 

anchore_engine/db/db_subscriptions.py

@@ -6,8 +6,10 @@
 
 
 def _compute_subscription_id(userId, subscription_key, subscription_type):
-    return hashlib.md5(
-        "+".join([userId, subscription_key, subscription_type]).encode("utf-8")
+    return hashlib.new(
+        "md5",
+        "+".join([userId, subscription_key, subscription_type]).encode("utf-8"),
+        usedforsecurity=False,
     ).hexdigest()
 
 

anchore_engine/db/entities/catalog.py

@@ -758,7 +758,7 @@ class ImageImportOperation(Base, UtilMixin):
     __tablename__ = "image_imports"
 
     uuid = Column(String, primary_key=True, default=anchore_uuid)
-    account = Column(String, index=True)
+    account = Column(String)
     expires_at = Column(DateTime)
     status = Column(Enum(ImportState))
     created_at = Column(DateTime, default=anchore_now_datetime)
@@ -767,6 +767,8 @@ class ImageImportOperation(Base, UtilMixin):
     )
     contents = relationship("ImageImportContent", back_populates="operation")
 
+    __table_args__ = (Index("ix_ae_image_imports_account", account), {})
+
     def to_json(self):
         j = super().to_json()
         j["status"] = self.status.value

anchore_engine/db/entities/identity.py

@@ -3,7 +3,7 @@
 
 from authlib.integrations.sqla_oauth2.client_mixin import OAuth2ClientMixin
 from authlib.integrations.sqla_oauth2.tokens_mixins import TokenMixin
-from sqlalchemy import Boolean, Column, Enum, ForeignKey, Integer, String, Text
+from sqlalchemy import Boolean, Column, Enum, ForeignKey, Integer, String, Text, Index
 from sqlalchemy.orm import relationship
 
 from anchore_engine.db.entities.common import Base, UtilMixin, anchore_now, anchore_uuid
@@ -79,16 +79,14 @@ class AccountUser(Base, UtilMixin):
     __tablename__ = "account_users"
 
     username = Column(String, primary_key=True)  # Enforce globally unique user names
-    account_name = Column(String, ForeignKey(Account.name), index=True)
+    account_name = Column(String, ForeignKey(Account.name))
     type = Column(
         Enum(UserTypes, name="user_types"), nullable=False, default=UserTypes.native
     )
     source = Column(String)
     created_at = Column(Integer, default=anchore_now)
     last_updated = Column(Integer, default=anchore_now)
-    uuid = Column(
-        "uuid", String, unique=True, nullable=False, default=anchore_uuid, index=True
-    )
+    uuid = Column("uuid", String, unique=True, nullable=False, default=anchore_uuid)
 
     account = relationship(
         "Account", back_populates="users", lazy="joined", innerjoin=True
@@ -100,6 +98,11 @@ class AccountUser(Base, UtilMixin):
         cascade="all, delete-orphan",
     )
 
+    __table_args__ = (
+        Index("ix_ae_account_users_account_name", account_name),
+        Index("ix_ae_account_users_uuid", uuid),
+    )
+
     def to_dict(self):
         """
         Override the base imple to include credentials
@@ -158,12 +161,14 @@ class OAuth2Token(Base, UtilMixin, TokenMixin):
     client_id = Column(String)
     token_type = Column(String)
     access_token = Column(String, unique=True, nullable=False)
-    refresh_token = Column(String, index=True)
+    refresh_token = Column(String)
     scope = Column(Text, default="")
     revoked = Column(Boolean, default=False)
     issued_at = Column(Integer, nullable=False, default=lambda: int(time.time()))
     expires_in = Column(Integer, nullable=False, default=0)
 
+    __table_args__ = (Index("ix_ae_oauth2_tokens_refresh_token", refresh_token), {})
+
     def get_scope(self):
         return self.scope
 

anchore_engine/db/entities/policy_engine.py

@@ -151,7 +151,7 @@ class GrypeDBFeedMetadata(Base):
     __tablename__ = "grype_db_feed_metadata"
 
     archive_checksum = Column(String, primary_key=True)
-    db_checksum = Column(String, nullable=True, index=True)
+    db_checksum = Column(String, nullable=True)
     schema_version = Column(String, nullable=False)
     object_url = Column(String, nullable=False)
     active = Column(Boolean, nullable=False)
@@ -166,6 +166,11 @@ class GrypeDBFeedMetadata(Base):
     synced_at = Column(DateTime, nullable=True)
     groups = Column(JSONB, default=[])
 
+    __table_args__ = (
+        Index("ix_ae_grype_db_feed_metadata_db_checksum", db_checksum),
+        {},
+    )
+
 
 class GenericFeedDataRecord(Base):
     """
@@ -843,7 +848,6 @@ class NvdV2Metadata(Base):
             name="vulnerability_severities",
         ),
         nullable=False,
-        index=True,
     )
     description = Column(String, nullable=True)
     cvss_v2 = Column(JSON, nullable=True)
@@ -860,6 +864,11 @@ class NvdV2Metadata(Base):
         DateTime, default=datetime.datetime.utcnow, onupdate=datetime.datetime.utcnow
     )
 
+    __table_args__ = (
+        Index("ix_ae_feed_data_nvdv2_vulnerabilities_severity", severity),
+        {},
+    )
+
     def __repr__(self):
         return "<{} name={}, created_at={}>".format(
             self.__class__, self.name, self.created_at
@@ -1083,7 +1092,6 @@ class VulnDBMetadata(Base):
             name="vulnerability_severities",
         ),
         nullable=False,
-        index=True,
     )
     title = Column(String, nullable=True)
     description = Column(String, nullable=True)
@@ -1105,6 +1113,11 @@ class VulnDBMetadata(Base):
         DateTime, default=datetime.datetime.utcnow, onupdate=datetime.datetime.utcnow
     )
 
+    __table_args__ = (
+        Index("ix_ae_feed_data_vulndb_vulnerabilities_severity", severity),
+        {},
+    )
+
     def __repr__(self):
         return "<{} name={}, created_at={}>".format(
             self.__class__, self.name, self.created_at
@@ -3378,8 +3391,8 @@ class ImageVulnerabilitiesReport(Base, StorageInterface):
 
     account_id = Column(String, primary_key=True)
     image_digest = Column(String, primary_key=True)
-    report_key = Column(String, index=True)
-    generated_at = Column(DateTime, index=True)
+    report_key = Column(String)
+    generated_at = Column(DateTime)
     result = Column(JSONB)
     created_at = Column(DateTime, default=datetime.datetime.utcnow)
     last_modified = Column(
@@ -3388,6 +3401,11 @@ class ImageVulnerabilitiesReport(Base, StorageInterface):
         onupdate=datetime.datetime.utcnow,
     )
 
+    __table_args__ = (
+        Index("ix_ae_image_vulnerabilities_reports_report_key", report_key),
+        Index("ix_ae_image_vulnerabilities_reports_generated_at", generated_at),
+    )
+
 
 class CachedPolicyEvaluation(Base, StorageInterface):
     __tablename__ = "policy_engine_evaluation_cache"

anchore_engine/services/apiext/api/controllers/policies.py

@@ -163,8 +163,10 @@ def add_policy(bundle):
         if "id" in jsondata and jsondata["id"]:
             policyId = jsondata["id"]
         else:
-            policyId = hashlib.md5(
-                str(userId + ":" + jsondata["name"]).encode("utf8")
+            policyId = hashlib.new(
+                "md5",
+                str(userId + ":" + jsondata["name"]).encode("utf8"),
+                usedforsecurity=False,
             ).hexdigest()
             jsondata["id"] = policyId
 

anchore_engine/services/catalog/catalog_impl.py

@@ -1754,10 +1754,12 @@ def perform_policy_evaluation(
                 curr_evaluation_result["status"] = "fail"
 
         # set up the newest evaluation
-        evalId = hashlib.md5(
+        evalId = hashlib.new(
+            "md5",
             ":".join(
-                [policyId, userId, imageDigest, fulltag, str(curr_final_action)]
-            ).encode("utf8")
+                [policyId, userId, imageDigest, fulltag, str(curr_final_action)],
+            ).encode("utf8"),
+            usedforsecurity=False,
         ).hexdigest()
         curr_evaluation_record = anchore_engine.common.helpers.make_eval_record(
             userId,