Merge branch 'master' into akamai-poc

.github/workflows/CI.yaml

@@ -100,7 +100,7 @@ jobs:
       run: tox -e py39-unit -- --cov=./ --cov-report=xml
 
     - name: Upload coverage reports to Codecov
-      uses: codecov/codecov-action@v3
+      uses: codecov/codecov-action@v4
       with:
         token: ${{ secrets.CODECOV_TOKEN }}
         flags: unit
@@ -203,9 +203,9 @@ jobs:
 
     - name: Start Quay
       run: |
-        docker-compose up -d redis quay-db
+        docker compose up -d redis quay-db
         docker exec -t quay-db bash -c 'while ! pg_isready; do echo "waiting for postgres"; sleep 2; done'
-        DOCKER_USER="1001:0" docker-compose up -d --no-build quay
+        DOCKER_USER="1001:0" docker compose up -d --no-build quay
 
     - name: Checkout
       uses: actions/checkout@v3
@@ -259,6 +259,19 @@ jobs:
         name: cypress-videos
         path: web/cypress/videos
 
+    - name: Create report
+      run: |
+        mkdir -p .logs/
+        docker logs quay-quay >.logs/quay.log 2>&1 || true
+      if: always()
+
+    - name: Upload Quay logs
+      uses: actions/upload-artifact@v3
+      with:
+        name: logs
+        path: .logs/
+      if: always()
+
   mysql:
     name: E2E MySQL Test
     runs-on: ubuntu-20.04

.github/workflows/build-and-publish.yaml

@@ -30,7 +30,7 @@ jobs:
         run: |
           BRANCH_NAME=${GITHUB_REF#refs/heads/}
           echo "version=${BRANCH_NAME/${{ env.BRANCH_PREFIX }}/}" >> $GITHUB_OUTPUT
-      
+
       - name: Setup SSH config for builders
         env:
           BUILDER_AARCH64_SSH_CONFIG: ${{ secrets.BUILDER_AARCH64_SSH_CONFIG }}
@@ -52,11 +52,11 @@ jobs:
           touch ~/.ssh/id_builder_ppc64le
           chmod 600 ~/.ssh/id_builder_ppc64le
           echo "$BUILDER_PPC64LE_SSH_KEY" >~/.ssh/id_builder_ppc64le
-          
+
           touch ~/.ssh/id_builder_s390x
           chmod 600 ~/.ssh/id_builder_s390x
           echo "$BUILDER_S390X_SSH_KEY" > ~/.ssh/id_builder_s390x
-          
+
           touch ~/.ssh/known_hosts
           chmod 600 ~/.ssh/known_hosts
           cat >~/.ssh/known_hosts <<END
@@ -74,11 +74,11 @@ jobs:
           Host builder-ppc64le
             IdentityFile "~/.ssh/id_builder_ppc64le"
           $BUILDER_PPC64LE_SSH_CONFIG
-          
+
           Host builder-s390x
             StrictHostKeyChecking no
             HostName $BUILDER_S390X_SSH_HOST
-            User root
+            User wfuser
             IdentityFile "~/.ssh/id_builder_s390x"
           END
 
@@ -110,4 +110,3 @@ jobs:
           platforms: linux/amd64,linux/arm64,linux/ppc64le,linux/s390x
           push: true
           tags: ${{ env.REGISTRY }}/${{ env.REPO_NAME }}:${{ github.event.inputs.tag || env.TAG }}
-

.github/workflows/config-tool.yaml

@@ -22,7 +22,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@v4
         with:
-          go-version: ^1.19
+          go-version: ^1.21
           cache-dependency-path: config-tool/go.sum
 
       - name: Verify go.mod
@@ -35,7 +35,7 @@ jobs:
       - name: Set up Go 1.x
         uses: actions/setup-go@v4
         with:
-          go-version: ^1.19
+          go-version: ^1.21
           cache-dependency-path: config-tool/go.sum
 
       - name: Checkout
@@ -47,7 +47,7 @@ jobs:
   tests:
     name: Tests
     runs-on: ubuntu-latest
-    container: docker.io/library/golang:1.19-buster
+    container: docker.io/library/golang:1.21
     services:
       postgres:
         image: postgres:11.5
@@ -65,8 +65,6 @@ jobs:
         ports:
           - 3306:3306
         options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=5
-      jwt:
-        image: quay.io/coreos/jwt-auth-example:latest
     steps:
       - name: Checkout
         uses: actions/checkout@v3

.github/workflows/prepare-release.yml

@@ -10,7 +10,7 @@ on:
         default: 'master'
       tag:
         description: 'the tag to be released'
-        rdequired: true
+        required: true
       jira:
         description: 'PROJQUAY jira release issue'
         required: true
@@ -33,6 +33,11 @@ jobs:
           chmod u+x /tmp/git-chglog
           echo "creating change log for tag: ${{ github.event.inputs.tag }}"
           /tmp/git-chglog --next-tag "${{ github.event.inputs.tag }}" --tag-filter-pattern "v3.*" -o CHANGELOG.md v3.6.0-alpha.4..
+      - name: Modify changelog URL to point to current y-stream
+        run: |
+          echo "modifying changelog URL to point to current y-stream release."
+          QUAY_VERSION=$(echo ${{ github.event.inputs.branch }} | cut -d'-' -f2)
+          sed -in --regexp-extended "s/\/[0-9\.]+\//\/$QUAY_VERSION\//" CHANGELOG.md
       - name: Create Pull Request
         uses: peter-evans/[email protected]
         with:

.gitignore

@@ -1,5 +1,6 @@
 *.pyc
 venv
+.venv
 screenshots/screenshots/
 conf/stack
 */node_modules

Makefile

@@ -1,6 +1,6 @@
 SHELL := /bin/bash
 DOCKER ?= docker
-DOCKER_COMPOSE ?= $(DOCKER)-compose
+DOCKER_COMPOSE ?= $(DOCKER) compose
 
 export PATH := ./venv/bin:$(PATH)
 

README.md

@@ -16,19 +16,18 @@ High-level features include:
 
 - Docker Registry Protocol [v2]
 - Docker Manifest Schema [v2.1], [v2.2]
-- [AppC Image Discovery] via on-demand transcoding
-- Image Squashing via on-demand transcoding
+- OCI spec v1.1 support [oci]
 - Authentication provided by [LDAP], [Keystone], [OIDC], [Google], and [GitHub]
 - ACLs, team management, and auditability logs
-- Geo-replicated storage provided by local filesystems, [S3], [GCS], [Swift], and [Ceph]
+- Geo-replicated storage provided by local filesystems, [S3], [GCS], [Swift], [Ceph] and [ODF]
 - Continuous Integration integrated with [GitHub], [Bitbucket], [GitLab], and [git]
 - Security Vulnerability Analysis via [Clair]
 - [Swagger]-compliant HTTP API
 
 [v2]: https://docs.docker.com/registry/spec/api/
 [v2.1]: https://github.com/docker/distribution/blob/master/docs/spec/manifest-v2-1.md
 [v2.2]: https://github.com/docker/distribution/blob/master/docs/spec/manifest-v2-2.md
-[AppC Image Discovery]: https://github.com/appc/spec/blob/master/spec/discovery.md
+[oci]: https://github.com/opencontainers/distribution-spec/tree/v1.1.0-rc1?tab=readme-ov-file#oci-distribution-specification
 [LDAP]: https://en.wikipedia.org/wiki/Lightweight_Directory_Access_Protocol
 [Keystone]: http://docs.openstack.org/developer/keystone
 [OIDC]: https://en.wikipedia.org/wiki/OpenID_Connect
@@ -38,6 +37,7 @@ High-level features include:
 [GCS]: https://cloud.google.com/storage
 [Swift]: http://swift.openstack.org
 [Ceph]: http://docs.ceph.com/docs/master/radosgw/config
+[ODF]: https://www.redhat.com/en/technologies/cloud-computing/openshift-data-foundation
 [GitHub]: https://github.com
 [Bitbucket]: https://bitbucket.com
 [GitLab]: https://gitlab.com
@@ -63,13 +63,13 @@ High-level features include:
 
 * Mailing List: [[email protected]]
 * IRC: #quay on [libera.chat]
-* Bug tracking: [JBoss JIRA]
+* Bug tracking: [Red Hat JIRA]
 * Security Issues: [[email protected]]
 * Community meetings held the first Wednesday of every month 11:00 AM EST: [meeting link](https://meet.google.com/rsj-atvk-zth)
 
 [[email protected]]: https://groups.google.com/forum/#!forum/quay-sig
 [libera.chat]: https://web.libera.chat/?channel=#quay
-[JBoss JIRA]: https://issues.jboss.org/projects/PROJQUAY
+[Red Hat JIRA]: https://issues.redhat.com/projects/PROJQUAY
 [[email protected]]: mailto:[email protected]
 
 ## License

app.py

@@ -252,7 +252,6 @@ def _request_end(resp):
 sentry = Sentry(app)
 build_logs = BuildLogs(app)
 userevents = UserEventsBuilderModule(app)
-instance_keys = InstanceKeys(app)
 label_validator = LabelValidator(app)
 build_canceller = BuildCanceller(app)
 

auth/credentials.py

@@ -2,9 +2,10 @@
 from enum import Enum
 
 from flask import request
+from jwt import InvalidTokenError
 
 import features
-from app import app, authentication
+from app import app, authentication, instance_keys
 from auth.credential_consts import (
     ACCESS_TOKEN_USERNAME,
     APP_SPECIFIC_TOKEN_USERNAME,
@@ -119,10 +120,9 @@ def validate_credentials(auth_username, auth_password_or_token):
     if is_robot:
         logger.debug("Found credentials header for robot %s", auth_username)
         try:
-            robot = model.user.verify_robot(auth_username, auth_password_or_token)
-
+            robot = model.user.verify_robot(auth_username, auth_password_or_token, instance_keys)
+            assert robot
             logger.debug("Successfully validated credentials for robot %s", auth_username)
-
             return ValidateResult(AuthKind.credentials, robot=robot), CredentialKind.robot
         except model.DeactivatedRobotOwnerException as dre:
             robot_owner, robot_name = parse_robot_username(auth_username)
@@ -155,7 +155,7 @@ def validate_credentials(auth_username, auth_password_or_token):
                 ValidateResult(AuthKind.credentials, error_message=str(dre)),
                 CredentialKind.robot,
             )
-        except model.InvalidRobotCredentialException as ire:
+        except (model.InvalidRobotCredentialException, InvalidTokenError) as ire:
             logger.debug("Failed to validate credentials for robot %s: %s", auth_username, ire)
 
             if app.config.get("ACTION_LOG_AUDIT_LOGIN_FAILURES"):

auth/decorators.py

@@ -10,6 +10,7 @@
 from auth.signedgrant import validate_signed_grant
 from auth.validateresult import AuthKind
 from util.http import abort
+from util.security.federated_robot_auth import validate_federated_auth
 
 logger = logging.getLogger(__name__)
 
@@ -77,6 +78,7 @@ def wrapper(*args, **kwargs):
 process_auth_or_cookie = _auth_decorator(handlers=[validate_basic_auth, validate_session_cookie])
 process_basic_auth = _auth_decorator(handlers=[validate_basic_auth], pass_result=True)
 process_basic_auth_no_pass = _auth_decorator(handlers=[validate_basic_auth])
+process_federated_auth = _auth_decorator(handlers=[validate_federated_auth], pass_result=True)
 
 
 def require_session_login(func):

auth/oauth.py

@@ -14,9 +14,9 @@
     _conduct_oauth_login,
     get_jwt_issuer,
     get_sub_username_email_from_token,
-    is_jwt,
 )
 from oauth.oidc import PublicKeyLoadException
+from util.security.jwtutil import is_jwt
 
 logger = logging.getLogger(__name__)
 

auth/test/mock_oidc_server.py

@@ -0,0 +1,116 @@
+# Mock OIDC discovery and token endpoint data
+import datetime
+import json
+import uuid
+
+import jwt
+
+MOCK_DISCOVERY_RESPONSE = {
+    "issuer": "https://mock-oidc-server.com",
+    "authorization_endpoint": "https://mock-oidc-server.com/authorize",
+    "token_endpoint": "https://mock-oidc-server.com/token",
+    "jwks_uri": "https://mock-oidc-server.com/.well-known/jwks.json",
+    "userinfo_endpoint": "https://mock-oidc-server.com/userinfo",
+    "response_types_supported": ["code", "id_token", "token id_token"],
+    "subject_types_supported": ["public"],
+    "id_token_signing_alg_values_supported": ["RS256"],
+}
+
+MOCK_PRIVATE_KEY = """
+-----BEGIN RSA PRIVATE KEY-----
+MIICXwIBAAKBgQDHd3NJdianKlLgzUmuc/fqYr/xFEDV7Ud3bPnO1N2r5UST7Rlj
+XkY2aEf6EL/4FvFZlKW/W6vwFelPMuAZGlZR717IABtj2YLpH8HnO53HqofezZHw
+QsahHwxmPJLXAl7Q4sdEg+/06bzsrFlYPWBftWpWKtUiPPK2KtmGdPFEEQIDAQAB
+AoGBAKIYNj36oAq04EkDSt9UKqH0wdqeBNpUSwGIM7GbVtD8LbCwuzL/R7urHuLe
+fcKUkmmj3NYXHzCp/cF4rJh5yK6317oim3MJjELYyY9K8eAZ2QRO/66JhphZqOD0
+XJ6iYqxvX62vxqoixvlXDhWLm3Gtv/57dKGgy5jkjhZUYHphAkEA+haxmLvTKgDD
+9yDVOjv2iEPrn1IBDeYRrGcl4byZPzwXmtp7RuXtxdB1irtkoagdjySeYglIdOJ6
++EqKtP/bPQJBAMwucEeQYAHaIHFpYORaY+VlgCT97gcj08BHZByDm5YA0oQxIi+W
+jMz0NCdDT9eqUAGszZ6T5PvsOtnFvPOfKWUCQQDzujYuwa4UG1bge7ES5eln97mk
+NYktgHDs8kGq8+DuDaR7mD3YZLELvhMvt11lZrAYFvn8VUu2DhsF66+uokOJAkEA
+vw14/E2ouDLthpFvG11E+iJWnMaKUl4AxntGvrObAuo0EYOUFGlPyHt8zXxbmlZ/
+1IFoSUjjy6KIkrtHCcLVTQJBAJB0NIhj1E8PdES5+s9XfqnMttK4V8lc46bb/3+U
+2H0hVBT7vR5sr+QjzEYSATW14c/9QBskZgsbtSEz6zf9+qU=
+-----END RSA PRIVATE KEY-----
+"""
+
+MOCK_PUBLIC_KEY = """
+-----BEGIN PUBLIC KEY-----
+MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDHd3NJdianKlLgzUmuc/fqYr/x
+FEDV7Ud3bPnO1N2r5UST7RljXkY2aEf6EL/4FvFZlKW/W6vwFelPMuAZGlZR717I
+ABtj2YLpH8HnO53HqofezZHwQsahHwxmPJLXAl7Q4sdEg+/06bzsrFlYPWBftWpW
+KtUiPPK2KtmGdPFEEQIDAQAB
+-----END PUBLIC KEY-----
+"""
+
+MOCK_JWKS_RESPONSE = {
+    "keys": [
+        {
+            "kty": "RSA",
+            "n": "x3dzSXYmpypS4M1JrnP36mK_8RRA1e1Hd2z5ztTdq-VEk-0ZY15GNmhH-hC_-BbxWZSlv1ur8BXpTzLgGRpWUe9eyAAbY9mC6R_B5zudx6qH3s2R8ELGoR8MZjyS1wJe0OLHRIPv9Om87KxZWD1gX7VqVirVIjzytirZhnTxRBE",
+            "e": "AQAB",
+            "kid": "mock-key-id",
+        }
+    ]
+}
+
+
+# Mock for discovery, JWKS, and token endpoints
+def mock_get(obj, url, *args, **kwargs):
+    if url == "https://mock-oidc-server.com/.well-known/openid-configuration":
+        return MockResponse(MOCK_DISCOVERY_RESPONSE, 200)
+    elif url == "https://mock-oidc-server.com/.well-known/jwks.json":
+        return MockResponse(MOCK_JWKS_RESPONSE, 200)
+    return MockResponse({}, 404)
+
+
+def mock_request(obj, method, url, *args, **kwargs):
+    return mock_get(None, url, *args, **kwargs)
+
+
+class MockResponse:
+    def __init__(self, json_data, status_code):
+        self.json_data = json_data
+        self.status_code = status_code
+
+    def json(self):
+        return self.json_data
+
+    @property
+    def text(self):
+        return json.dumps(self.json_data)
+
+
+def generate_mock_oidc_token(
+    issuer="https://mock-oidc-server.com",
+    subject="mock-subject",
+    audience="mock-client-id",
+    expiry_seconds=3600,
+    issued_at=None,
+):
+    now = datetime.datetime.now()
+    iat = now - datetime.timedelta(seconds=30)
+    if issued_at is not None:
+        iat = issued_at
+
+    exp = iat + datetime.timedelta(seconds=expiry_seconds)
+
+    payload = {
+        "iss": issuer,
+        "sub": subject,
+        "aud": audience,
+        "exp": int(exp.timestamp()),
+        "iat": int(iat.timestamp()),
+        "nbf": int(iat.timestamp()),
+        "nonce": str(uuid.uuid4()),
+        "name": "Mock User",
+        "preferred_username": "mockuser",
+        "given_name": "Mock",
+        "family_name": "User",
+        "email": "[email protected]",
+        "email_verified": True,
+    }
+
+    headers = {"kid": "mock-key-id"}
+
+    return jwt.encode(payload, MOCK_PRIVATE_KEY, algorithm="RS256", headers=headers)