Ensure tests are failing CI (#840)
This commit is contained in:
parent
fdcb38537f
commit
c54a5ac277
3 changed files with 105 additions and 37 deletions
5
Makefile
5
Makefile
|
|
@ -1,5 +1,8 @@
|
||||||
.PHONY: help build test test-unit test-integration test-endpoints test-publish test-all lint lint-fix validate validate-schemas validate-examples check ko-build ko-rebuild dev-compose dev-down clean publisher generate-schema check-schema
|
.PHONY: help build test test-unit test-integration test-endpoints test-publish test-all lint lint-fix validate validate-schemas validate-examples check ko-build ko-rebuild dev-compose dev-down clean publisher generate-schema check-schema
|
||||||
|
|
||||||
|
# Use bash for all commands to support pipefail
|
||||||
|
SHELL := /bin/bash
|
||||||
|
|
||||||
# Default target
|
# Default target
|
||||||
help: ## Show this help message
|
help: ## Show this help message
|
||||||
@echo "Available targets:"
|
@echo "Available targets:"
|
||||||
|
|
@ -33,7 +36,7 @@ test-unit: ## Run unit tests with coverage (requires PostgreSQL)
|
||||||
@sleep 3
|
@sleep 3
|
||||||
@echo ""
|
@echo ""
|
||||||
@echo "Running unit tests..."
|
@echo "Running unit tests..."
|
||||||
@if command -v gotestsum >/dev/null 2>&1; then \
|
@set -o pipefail; if command -v gotestsum >/dev/null 2>&1; then \
|
||||||
gotestsum --format pkgname-and-test-fails -- -race -coverprofile=coverage.out -covermode=atomic ./internal/... ./cmd/... 2>&1 | grep -v "ld: warning:"; \
|
gotestsum --format pkgname-and-test-fails -- -race -coverprofile=coverage.out -covermode=atomic ./internal/... ./cmd/... 2>&1 | grep -v "ld: warning:"; \
|
||||||
else \
|
else \
|
||||||
go test -race -coverprofile=coverage.out -covermode=atomic ./internal/... ./cmd/... 2>&1 | grep -v "ld: warning:" | grep -v "^ld:"; \
|
go test -race -coverprofile=coverage.out -covermode=atomic ./internal/... ./cmd/... 2>&1 | grep -v "ld: warning:" | grep -v "^ld:"; \
|
||||||
|
|
|
||||||
|
|
@ -50,8 +50,9 @@ func TestOpenAPIEndpointCompliance(t *testing.T) {
|
||||||
BuildTime: "test",
|
BuildTime: "test",
|
||||||
}
|
}
|
||||||
|
|
||||||
// Register V0 routes exactly like production does
|
// Register V0 and V0.1 routes exactly like production does
|
||||||
router.RegisterV0Routes(api, cfg, nil, nil, versionInfo) // nil service and metrics for schema testing
|
router.RegisterV0Routes(api, cfg, nil, nil, versionInfo) // nil service and metrics for schema testing
|
||||||
|
router.RegisterV0_1Routes(api, cfg, nil, nil, versionInfo) // Register v0.1 routes for compliance
|
||||||
|
|
||||||
// Get the OpenAPI schema
|
// Get the OpenAPI schema
|
||||||
req := httptest.NewRequest(http.MethodGet, "/openapi.yaml", nil)
|
req := httptest.NewRequest(http.MethodGet, "/openapi.yaml", nil)
|
||||||
|
|
|
||||||
|
|
@ -17,52 +17,56 @@ func TestValidateOCI_RegistryAllowlist(t *testing.T) {
|
||||||
identifier string
|
identifier string
|
||||||
expectError bool
|
expectError bool
|
||||||
errorMsg string
|
errorMsg string
|
||||||
|
mustNotContainMsg string // Error message that must NOT appear (for allowed registries)
|
||||||
}{
|
}{
|
||||||
// Allowed registries - use real public images that exist
|
// Allowed registries - use real public images that exist
|
||||||
// These should fail with "missing required annotation" (no MCP label)
|
// These should fail with annotation-related errors (missing or mismatched)
|
||||||
// NOT with "unsupported registry", "does not exist", or "is private" errors
|
// NOT with "unsupported registry", "does not exist", or "is private" errors
|
||||||
{
|
{
|
||||||
name: "Docker Hub should be allowed",
|
name: "Docker Hub should be allowed",
|
||||||
identifier: "docker.io/library/alpine:latest",
|
identifier: "docker.io/library/alpine:latest",
|
||||||
expectError: true,
|
expectError: true,
|
||||||
errorMsg: "missing required annotation",
|
errorMsg: "missing required annotation",
|
||||||
|
mustNotContainMsg: "unsupported OCI registry",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "Docker Hub without explicit registry should default and be allowed",
|
name: "Docker Hub without explicit registry should default and be allowed",
|
||||||
identifier: "library/hello-world:latest",
|
identifier: "library/hello-world:latest",
|
||||||
expectError: true,
|
expectError: true,
|
||||||
errorMsg: "missing required annotation",
|
errorMsg: "missing required annotation",
|
||||||
|
mustNotContainMsg: "unsupported OCI registry",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "GHCR should be allowed",
|
name: "GHCR should be allowed",
|
||||||
identifier: "ghcr.io/containerbase/base:latest",
|
identifier: "ghcr.io/containerbase/base:latest",
|
||||||
expectError: true,
|
expectError: true,
|
||||||
errorMsg: "missing required annotation",
|
errorMsg: "missing required annotation",
|
||||||
|
mustNotContainMsg: "unsupported OCI registry",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "Artifact Registry regional should be allowed",
|
name: "Artifact Registry regional should be allowed",
|
||||||
identifier: "us-central1-docker.pkg.dev/database-toolbox/toolbox/toolbox:latest",
|
identifier: "us-central1-docker.pkg.dev/database-toolbox/toolbox/toolbox:latest",
|
||||||
expectError: true,
|
expectError: true,
|
||||||
errorMsg: "missing required annotation",
|
// This image has an MCP annotation but with a different server name,
|
||||||
|
// so we get "ownership validation failed" instead of "missing required annotation"
|
||||||
|
// Both are acceptable - what matters is the registry is allowed
|
||||||
|
mustNotContainMsg: "unsupported OCI registry",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "Artifact Registry multi-region should be allowed",
|
name: "Artifact Registry multi-region should be allowed",
|
||||||
identifier: "us-docker.pkg.dev/berglas/berglas/berglas:latest",
|
identifier: "us-docker.pkg.dev/berglas/berglas/berglas:latest",
|
||||||
expectError: true,
|
expectError: true,
|
||||||
errorMsg: "missing required annotation",
|
errorMsg: "missing required annotation",
|
||||||
|
mustNotContainMsg: "unsupported OCI registry",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "MCR should be allowed",
|
name: "MCR should be allowed",
|
||||||
identifier: "mcr.microsoft.com/dotnet/aspire-dashboard:9.5.0",
|
identifier: "mcr.microsoft.com/dotnet/aspire-dashboard:9.5.0",
|
||||||
expectError: true,
|
expectError: true,
|
||||||
errorMsg: "missing required annotation",
|
errorMsg: "missing required annotation",
|
||||||
|
mustNotContainMsg: "unsupported OCI registry",
|
||||||
},
|
},
|
||||||
{
|
// Removed ACR test with non-existent host - ACR support is tested elsewhere
|
||||||
name: "ACR should be allowed",
|
|
||||||
identifier: "azurearcjumpstart.azurecr.io/hello-arc:latest",
|
|
||||||
expectError: true,
|
|
||||||
errorMsg: "missing required annotation",
|
|
||||||
},
|
|
||||||
|
|
||||||
// Disallowed registries
|
// Disallowed registries
|
||||||
{
|
{
|
||||||
|
|
@ -114,8 +118,14 @@ func TestValidateOCI_RegistryAllowlist(t *testing.T) {
|
||||||
|
|
||||||
if tt.expectError {
|
if tt.expectError {
|
||||||
assert.Error(t, err)
|
assert.Error(t, err)
|
||||||
// Should contain the specific error message
|
// For allowed registries, verify they don't get rejected at the registry check
|
||||||
|
if tt.mustNotContainMsg != "" {
|
||||||
|
assert.NotContains(t, err.Error(), tt.mustNotContainMsg)
|
||||||
|
}
|
||||||
|
// If a specific error message is expected, check for it
|
||||||
|
if tt.errorMsg != "" {
|
||||||
assert.Contains(t, err.Error(), tt.errorMsg)
|
assert.Contains(t, err.Error(), tt.errorMsg)
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
assert.NoError(t, err)
|
assert.NoError(t, err)
|
||||||
}
|
}
|
||||||
|
|
@ -123,6 +133,60 @@ func TestValidateOCI_RegistryAllowlist(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestValidateOCI_RegistryPatterns(t *testing.T) {
|
||||||
|
// This test verifies registry pattern matching (wildcards like *.azurecr.io and *.pkg.dev)
|
||||||
|
// without relying on external images that may not exist
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
identifier string
|
||||||
|
shouldFail bool // true if should fail at registry allowlist check
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "ACR registry pattern should be allowed",
|
||||||
|
identifier: "myregistry.azurecr.io/test/image:latest",
|
||||||
|
shouldFail: false, // Registry is allowed, will fail later on missing annotation
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "Another ACR registry should be allowed",
|
||||||
|
identifier: "contoso.azurecr.io/app:v1",
|
||||||
|
shouldFail: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "Artifact Registry should be allowed",
|
||||||
|
identifier: "us-west1-docker.pkg.dev/project/repo/image:tag",
|
||||||
|
shouldFail: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "GCR should be rejected at registry check",
|
||||||
|
identifier: "gcr.io/project/image:latest",
|
||||||
|
shouldFail: true, // Should fail at registry allowlist check
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
pkg := model.Package{
|
||||||
|
RegistryType: model.RegistryTypeOCI,
|
||||||
|
Identifier: tt.identifier,
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
err := registries.ValidateOCI(ctx, pkg, "com.example/test")
|
||||||
|
|
||||||
|
// All test cases should error (either at registry check or annotation check)
|
||||||
|
assert.Error(t, err)
|
||||||
|
|
||||||
|
if tt.shouldFail {
|
||||||
|
// Should fail at the registry allowlist check
|
||||||
|
assert.Contains(t, err.Error(), "unsupported OCI registry")
|
||||||
|
} else {
|
||||||
|
// Should NOT fail at registry check (allowed registry)
|
||||||
|
assert.NotContains(t, err.Error(), "unsupported OCI registry")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestValidateOCI_RejectsOldFormat(t *testing.T) {
|
func TestValidateOCI_RejectsOldFormat(t *testing.T) {
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue