diff --git a/README.md b/README.md index 11d9b96..9422464 100644 --- a/README.md +++ b/README.md @@ -144,6 +144,7 @@ nullify auth switch |---------|-------------| | `nullify pentest` | Run DAST pentest scans (cloud or local via Docker) | | `nullify bughunt` | Cloud-based automated bug hunting | +| `nullify secrets scan-file` | Local, in-process secret scan of a single file or stdin buffer — no API call, no network. Powers the IDE extension's on-save scanning | ### CI/CD diff --git a/cmd/cli/cmd/root.go b/cmd/cli/cmd/root.go index b931447..cd8faa7 100644 --- a/cmd/cli/cmd/root.go +++ b/cmd/cli/cmd/root.go @@ -164,6 +164,10 @@ func init() { // scpm's /scpm/dependencies/analyze. Wired at top level (not under // apiCmd) so `nullify deps analyze` reads naturally in CI scripts. commands.RegisterDepsAnalyzeCommand(rootCmd, getAPIClient) + + // Local, in-process scanning: no API client, no network call, so it + // hangs off the top-level 'secrets' command rather than 'api secrets'. + commands.RegisterScanFileCommand(secretsCmd) } func setupLogger(ctx context.Context) (context.Context, error) { diff --git a/cmd/cli/cmd/secrets.go b/cmd/cli/cmd/secrets.go new file mode 100644 index 0000000..774f707 --- /dev/null +++ b/cmd/cli/cmd/secrets.go @@ -0,0 +1,19 @@ +package cmd + +import ( + "github.com/spf13/cobra" +) + +// secretsCmd is the top-level home for local, in-process secret scanning +// (scan-file). The generated, API-backed secrets commands (allowlists, +// findings, ...) stay under 'api secrets' via RegisterSecretsCommands -- +// this tree is for scanning that never calls the platform API. +var secretsCmd = &cobra.Command{ + Use: "secrets", + Short: "Local secrets scanning", + Long: "Local, in-process secret scanning. Scans run entirely on this machine; no file content is sent to the Nullify API.", +} + +func init() { + rootCmd.AddCommand(secretsCmd) +} diff --git a/internal/commands/scanfile.go b/internal/commands/scanfile.go new file mode 100644 index 0000000..386278a --- /dev/null +++ b/internal/commands/scanfile.go @@ -0,0 +1,92 @@ +// Hand-written, not generated: scan-file has no OpenAPI operation to +// generate a client from (it never calls the Nullify API — see +// internal/scanfile's package doc for why, and for what engine it wraps). +// It follows the same RegisterXxxCommand(parent, ...) shape as this +// package's other hand-written files (context_push.go, pentest_bridge.go). +package commands + +import ( + "encoding/json" + "fmt" + "io" + "os" + + "github.com/spf13/cobra" + + "github.com/nullify-platform/cli/internal/scanfile" +) + +// RegisterScanFileCommand adds the 'scan-file' subcommand to parent (the +// hand-written top-level 'secrets' command; see cmd/cli/cmd/secrets.go). +func RegisterScanFileCommand(parent *cobra.Command) { + var ( + flagStdin bool + flagPath string + flagRepository string + ) + + cmd := &cobra.Command{ + Use: "scan-file [path]", + Short: "Scan a single file (or stdin buffer) for secrets and print normalized findings", + Long: `Scan a single file for hardcoded secrets and print normalized findings as a +JSON array on stdout. All matching runs in-process: no network call is made +and file content never leaves the machine. + +This is the fast-local scan the IDE extension runs on save. It scans only +the given file (or the buffer piped on stdin via --stdin), never a whole +repo, and emits engine-agnostic findings. + +Examples: + # Scan a file on disk + nullify secrets scan-file ./config.go + + # Scan an unsaved editor buffer (the editor pipes the buffer, --path labels it) + cat buffer.go | nullify secrets scan-file --stdin --path ./config.go`, + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + var ( + content []byte + path string + err error + ) + + if flagStdin { + if flagPath == "" { + return fmt.Errorf("--path is required when using --stdin") + } + if len(args) > 0 { + return fmt.Errorf("cannot combine a positional path with --stdin") + } + content, err = io.ReadAll(cmd.InOrStdin()) + if err != nil { + return fmt.Errorf("failed to read stdin: %w", err) + } + path = flagPath + } else { + if len(args) != 1 { + return fmt.Errorf("exactly one file path is required (or use --stdin)") + } + path = args[0] + content, err = os.ReadFile(path) + if err != nil { + return fmt.Errorf("failed to read %s: %w", path, err) + } + } + + findings := scanfile.ScanContent(content, path, flagRepository) + + enc := json.NewEncoder(cmd.OutOrStdout()) + enc.SetIndent("", " ") + if err := enc.Encode(findings); err != nil { + return fmt.Errorf("failed to encode findings: %w", err) + } + return nil + }, + } + + cmd.Flags().BoolVar(&flagStdin, "stdin", false, "read the file content from stdin instead of disk") + cmd.Flags().StringVar(&flagPath, "path", "", "path to label the scanned content with (required with --stdin)") + cmd.Flags().StringVar(&flagRepository, "repository", "", "repository identifier folded into the correlation fingerprint") + + parent.AddCommand(cmd) +} diff --git a/internal/commands/scanfile_test.go b/internal/commands/scanfile_test.go new file mode 100644 index 0000000..ca9aa78 --- /dev/null +++ b/internal/commands/scanfile_test.go @@ -0,0 +1,105 @@ +package commands + +import ( + "bytes" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/spf13/cobra" + "github.com/stretchr/testify/require" + + "github.com/nullify-platform/cli/internal/scanfile" +) + +func newTestRoot() *cobra.Command { + root := &cobra.Command{Use: "root"} + RegisterScanFileCommand(root) + return root +} + +// fakeAWSKey builds an AWS-access-key-ID-shaped fixture via concatenation so +// the 20-character string never appears contiguous in this file's own +// source text and can't trip a secret scanner on this repo's own commits. +func fakeAWSKey() string { + return "AKIA" + "ABCDEFGHIJKLMNOP" +} + +func TestRegisterScanFileCommandAddsScanFile(t *testing.T) { + root := newTestRoot() + cmd, _, err := root.Find([]string{"scan-file"}) + require.NoError(t, err) + require.Equal(t, "scan-file", cmd.Name()) +} + +func TestScanFileCommandScansDiskFile(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "config.go") + content := "awsKey := \"" + fakeAWSKey() + "\"\n" + require.NoError(t, os.WriteFile(path, []byte(content), 0o600)) + + root := newTestRoot() + var stdout bytes.Buffer + root.SetOut(&stdout) + root.SetArgs([]string{"scan-file", path}) + + require.NoError(t, root.Execute()) + + var findings []scanfile.NormalizedFinding + require.NoError(t, json.Unmarshal(stdout.Bytes(), &findings)) + require.NotEmpty(t, findings) + require.Equal(t, path, findings[0].File) +} + +func TestScanFileCommandScansStdin(t *testing.T) { + content := "awsKey := \"" + fakeAWSKey() + "\"\n" + + root := newTestRoot() + var stdout bytes.Buffer + root.SetOut(&stdout) + root.SetIn(strings.NewReader(content)) + root.SetArgs([]string{"scan-file", "--stdin", "--path", "config.go", "--repository", "owner/repo"}) + + require.NoError(t, root.Execute()) + + var findings []scanfile.NormalizedFinding + require.NoError(t, json.Unmarshal(stdout.Bytes(), &findings)) + require.NotEmpty(t, findings) + require.Equal(t, "config.go", findings[0].File) +} + +func TestScanFileCommandRequiresPathWithStdin(t *testing.T) { + root := newTestRoot() + root.SetOut(&bytes.Buffer{}) + root.SetErr(&bytes.Buffer{}) + root.SetIn(strings.NewReader("content")) + root.SetArgs([]string{"scan-file", "--stdin"}) + + require.Error(t, root.Execute()) +} + +func TestScanFileCommandRejectsPathAndStdinTogether(t *testing.T) { + root := newTestRoot() + root.SetOut(&bytes.Buffer{}) + root.SetErr(&bytes.Buffer{}) + root.SetIn(strings.NewReader("content")) + root.SetArgs([]string{"scan-file", "--stdin", "--path", "a.go", "b.go"}) + + require.Error(t, root.Execute()) +} + +func TestScanFileCommandCleanFileProducesEmptyArray(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "main.go") + require.NoError(t, os.WriteFile(path, []byte("package main\n\nfunc main() {}\n"), 0o600)) + + root := newTestRoot() + var stdout bytes.Buffer + root.SetOut(&stdout) + root.SetArgs([]string{"scan-file", path}) + + require.NoError(t, root.Execute()) + require.Equal(t, "[]\n", stdout.String()) +} diff --git a/internal/scanfile/baseline_rules.go b/internal/scanfile/baseline_rules.go new file mode 100644 index 0000000..00b08c7 --- /dev/null +++ b/internal/scanfile/baseline_rules.go @@ -0,0 +1,107 @@ +package scanfile + +import "regexp" + +// baselineRules covers well-known, publicly documented credential formats +// that nullifyRules doesn't already handle (nullifyRules is Nullify's own +// additions layered on top of gitleaks' several-hundred-rule default set; +// this package doesn't vendor that default set -- see scanfile.go's package +// doc). It is not a claim of parity with gitleaks' breadth: it is a +// deliberately small, high-confidence set for the formats most likely to +// turn up in a file someone is actively editing. +func baselineRules() []Rule { + return []Rule{ + { + ID: "aws-access-key-id", + Regex: regexp.MustCompile(`\b((?:AKIA|ASIA|ABIA|ACCA)[A-Z2-7]{16})\b`), + Keywords: []string{"akia", "asia", "abia", "acca"}, + }, + { + ID: "aws-secret-access-key", + Regex: regexp.MustCompile(`(?i)aws[a-z0-9_.-]{0,20}?(?:secret|access)[a-z0-9_.-]{0,20}?key[a-z0-9_.-]{0,20}?\s*[:=]\s*['"]?([A-Za-z0-9/+=]{40})['"]?`), + Keywords: []string{"aws"}, + MinEntropy: 4.0, + }, + { + ID: "github-pat", + Regex: regexp.MustCompile(`\bgh[oprsu]_[A-Za-z0-9]{36,251}\b`), + Keywords: []string{"ghp_", "gho_", "ghr_", "ghs_", "ghu_"}, + }, + { + ID: "gitlab-pat", + Regex: regexp.MustCompile(`\bglpat-[A-Za-z0-9_-]{20}\b`), + Keywords: []string{"glpat-"}, + }, + { + ID: "slack-token", + Regex: regexp.MustCompile(`\bxox[baprs]-[A-Za-z0-9-]{10,72}\b`), + Keywords: []string{"xoxb-", "xoxa-", "xoxp-", "xoxr-", "xoxs-"}, + }, + { + ID: "slack-webhook-url", + Regex: regexp.MustCompile(`https://hooks\.slack\.com/services/T[A-Za-z0-9]{6,}/B[A-Za-z0-9]{6,}/[A-Za-z0-9]{16,}`), + Keywords: []string{"hooks.slack.com"}, + }, + { + ID: "stripe-secret-key", + Regex: regexp.MustCompile(`\bsk_live_[A-Za-z0-9]{24,}\b`), + Keywords: []string{"sk_live_"}, + }, + { + ID: "stripe-restricted-key", + Regex: regexp.MustCompile(`\brk_live_[A-Za-z0-9]{24,}\b`), + Keywords: []string{"rk_live_"}, + }, + { + ID: "google-api-key", + Regex: regexp.MustCompile(`\bAIza[A-Za-z0-9_-]{35}\b`), + Keywords: []string{"AIza"}, + }, + { + ID: "pem-private-key", + Regex: regexp.MustCompile(`-----BEGIN\s?((?:RSA|EC|DSA|OPENSSH|PGP)\s)?PRIVATE KEY(\sBLOCK)?-----`), + Keywords: []string{"private key"}, + }, + { + ID: "npm-access-token", + Regex: regexp.MustCompile(`\bnpm_[A-Za-z0-9]{36}\b`), + Keywords: []string{"npm_"}, + }, + { + ID: "twilio-api-key", + Regex: regexp.MustCompile(`\bSK[0-9a-fA-F]{32}\b`), + Keywords: []string{"twilio"}, + }, + { + ID: "sendgrid-api-key", + Regex: regexp.MustCompile(`\bSG\.[A-Za-z0-9_-]{22}\.[A-Za-z0-9_-]{43}\b`), + Keywords: []string{"sg."}, + }, + { + ID: "mailgun-api-key", + Regex: regexp.MustCompile(`\bkey-[a-f0-9]{32}\b`), + Keywords: []string{"mailgun", "key-"}, + }, + { + ID: "discord-bot-token", + Regex: regexp.MustCompile(`\b[MN][A-Za-z\d]{23}\.[\w-]{6}\.[\w-]{27}\b`), + Keywords: []string{"discord"}, + }, + { + ID: "azure-storage-connection-string", + Regex: regexp.MustCompile(`(?i)DefaultEndpointsProtocol=https?;AccountName=[A-Za-z0-9]+;AccountKey=[A-Za-z0-9+/=]{20,};`), + Keywords: []string{"accountkey="}, + }, + { + ID: "heroku-api-key", + Regex: regexp.MustCompile(`(?i)heroku[a-z0-9_.-]{0,20}?(?:api)?[a-z0-9_.-]{0,20}?key\s*[:=]\s*['"]?([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})['"]?`), + Keywords: []string{"heroku"}, + }, + { + ID: "generic-api-key", + Regex: regexp.MustCompile(`(?i)(?:secret|token|api[_-]?key|apikey|password|passwd|credential)\s*[:=]\s*['"]([A-Za-z0-9+/=_.-]{16,64})['"]`), + Keywords: []string{"secret", "token", "key", "password", "passwd", "credential"}, + MinEntropy: 4.0, + }, + } +} diff --git a/internal/scanfile/nullify_rules.go b/internal/scanfile/nullify_rules.go new file mode 100644 index 0000000..0925fbd --- /dev/null +++ b/internal/scanfile/nullify_rules.go @@ -0,0 +1,209 @@ +package scanfile + +import "regexp" + +// nullifyRules returns Nullify's own custom secret-detection rules, ported +// verbatim (same IDs, regexes, keywords and entropy floors) from +// secrets/internal/scanners/gitleaks/nullify-config.go's GetNullifyRules. +// That function has no engine dependency of its own — it is plain data — so +// it crosses into this public repo cleanly. Each rule's ID (via Rule.Title) +// is descriptive enough on its own; the source file's rule-by-rule +// commentary is kept only where it explains something the ID can't. +func nullifyRules() []Rule { + return []Rule{ + { + ID: "artifactory-api-token", + Regex: regexp.MustCompile(`(?:[^A-Za-z0-9/+]|^)(AKC[a-zA-Z0-9]{10,})(?:[^A-Za-z0-9/+]|$)`), + Keywords: []string{"artifactory", "api", "token", "akc"}, + }, + { + ID: "artifactory-password", + Regex: regexp.MustCompile(`(?:[^A-Za-z0-9/+]|^)(AP[0-9ABCDEF][a-zA-Z0-9]{8,})(?:[^A-Za-z0-9/+]|$)`), + Keywords: []string{"artifactory", "password", "ap"}, + }, + { + ID: "basic-auth", + Regex: regexp.MustCompile(`://[^:/\?\#\[\]@!\$\&'\(\)\*\+,;=\s]+:([^:/\?\#\[\]@!\$\&'\(\)\*\+,;=\s]+)@`), + Keywords: []string{"://", "http", "https", "ftp"}, + }, + { + ID: "cloudant-credentials", + Regex: regexp.MustCompile(`(?im)((?:https?\:\/\/)[\w\-]+\:([0-9a-f]{64})\@[\w\-]+\.cloudant\.com)|((?:https?\:\/\/)[\w\-]+\:([a-z]{24})\@[\w\-]+\.cloudant\.com)|(?:^|\W)(?:\[|)(?:"|\'|)(?:cloudant|cl|clou)(?:_|-|)(?:api|)(?:key|pwd|pw|password|pass|token)(?:"|\'|)(?:\]|)(?: *)(?:=|:|:=|=>| +|::)(?: *)(?:"|\'|)([0-9a-f]{64})(?:"|\'|)|(?:^|\W)(?:\[|)(?:"|\'|)(?:cloudant|cl|clou)(?:_|-|)(?:api|)(?:key|pwd|pw|password|pass|token)(?:"|\'|)(?:\]|)(?: *)(?:=|:|:=|=>| +|::)(?: *)(?:"|\'|)([a-z]{24})(?:"|\'|)`), + Keywords: []string{"cloudant", "ibm", "cloud"}, + }, + { + ID: "ibm-cloud-iam-key", + Regex: regexp.MustCompile(`(?im)(?:^|\W)(?:\[|)(?:"|\'|)(?:ibm(?:_|-|)cloud(?:_|-|)iam|cloud(?:_|-|)iam|ibm(?:_|-|)cloud|ibm(?:_|-|)iam|ibm|iam|cloud|)(?:_|-|)(?:api|)(?:_|-|)(?:key|pwd|password|pass|token)(?:"|\'|)(?:\]|)(?: *)(?:=|:|:=|=>| +|::)(?: *)(?:"|\'|)([a-zA-Z0-9_\-]{44}[^a-zA-Z0-9_\-])(?:"|\'|)`), + Keywords: []string{"ibm", "cloud", "iam", "key"}, + }, + { + ID: "ibm-cos-hmac-credentials", + Regex: regexp.MustCompile(`(?i)(?:^|\W)(?:\[|)(?:"|\'|)(?:(?:ibm)?[-_]?cos[-_]?(?:hmac)?|)(?:_|-|)(?:secret[-_]?(?:access)?[-_]?key)(?:"|\'|)(?:\]|)(?: *)(?:=|:|:=|=>| +|::)(?: *)(?:"|\'|)([a-f0-9]{48}[^a-f0-9])(?:"|\'|)`), + Keywords: []string{"ibm", "cos", "hmac", "secret"}, + }, + { + ID: "softlayer-api-key", + Regex: regexp.MustCompile(`(?im)(?:^|\W)(?:\[|)(?:"|\'|)(?:softlayer|sl)(?:_|-|)(?:api|)(?:_|-|)(?:key|pwd|password|pass|token)(?:"|\'|)(?:\]|)(?: *)(?:=|:|:=|=>| +|::)(?: *)(?:"|\'|)([a-z0-9]{64})(?:"|\'|)|(?:http|https)://api.softlayer.com/soap/(?:v3|v3.1)/([a-z0-9]{64})`), + Keywords: []string{"softlayer", "sl", "api", "key"}, + }, + { + ID: "anthropic-api-key", + Regex: regexp.MustCompile(`sk-ant-api03-[A-Za-z0-9_\-]{93}AA`), + Keywords: []string{"sk-ant-api03"}, + }, + { + ID: "openai-project-key", + Regex: regexp.MustCompile(`sk-proj-[A-Za-z0-9_\-]{40,}`), + Keywords: []string{"sk-proj-"}, + }, + { + ID: "gcp-oauth-client-secret", + Regex: regexp.MustCompile(`GOCSPX-[A-Za-z0-9_\-]{28}`), + Keywords: []string{"GOCSPX-"}, + }, + { + ID: "cloudflare-api-token", + Regex: regexp.MustCompile(`(?i)(?:cloudflare|cf)[\s_-]*(?:api)?[\s_-]*(?:token|key)\s*(?:=|:|:=)\s*['"]?([A-Za-z0-9_\-]{40})['"]?`), + Keywords: []string{"cloudflare", "cf_api"}, + MinEntropy: 3.5, + }, + { + ID: "vercel-api-token", + Regex: regexp.MustCompile(`vc_[A-Za-z0-9]{24}`), + Keywords: []string{"vc_"}, + }, + { + ID: "flyio-api-token", + Regex: regexp.MustCompile(`FlyV1\s+fm2_[A-Za-z0-9_\-/+=]{100,}`), + Keywords: []string{"FlyV1", "fm2_"}, + }, + { + ID: "tailscale-api-key", + Regex: regexp.MustCompile(`tskey-(?:api|auth|client)-[A-Za-z0-9]{10,}-[A-Za-z0-9]{20,}`), + Keywords: []string{"tskey-"}, + }, + { + // Two shapes: st..<32 hex>.<32 hex>. + ID: "infisical-api-token", + Regex: regexp.MustCompile(`st\.(?:[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}|[0-9a-f]{24})\.[0-9a-f]{32}\.[0-9a-f]{32}`), + Keywords: []string{"st."}, + }, + { + ID: "onepassword-service-account-token", + Regex: regexp.MustCompile(`ops_eyJ[A-Za-z0-9+/=_\-]{200,}`), + Keywords: []string{"ops_eyJ"}, + }, + { + ID: "railway-api-token", + Regex: regexp.MustCompile(`(?i)railway[\s_-]*(?:api)?[\s_-]*(?:token|key)\s*(?:=|:|:=)\s*['"]?([a-f0-9\-]{36})['"]?`), + Keywords: []string{"railway"}, + MinEntropy: 3.0, + }, + { + ID: "postmark-server-token", + Regex: regexp.MustCompile(`(?i)postmark[\s_-]*(?:server)?[\s_-]*(?:token|key|api)\s*(?:=|:|:=)\s*['"]?([a-f0-9\-]{36})['"]?`), + Keywords: []string{"postmark"}, + }, + { + ID: "neon-database-url", + Regex: regexp.MustCompile(`postgres(?:ql)?://[^:]+:[^@]+@[^/]*\.neon\.tech`), + Keywords: []string{"neon.tech"}, + }, + { + ID: "supabase-pat", + Regex: regexp.MustCompile(`sbp_[a-f0-9]{40}`), + Keywords: []string{"sbp_"}, + }, + // AI provider keys absent from the gitleaks defaults. Regexes are + // taken verbatim from trufflehog's pkg/detectors//.go + // (per the internal rule table this was ported from). + { + ID: "groq-api-key", + Regex: regexp.MustCompile(`\b(gsk_[a-zA-Z0-9]{52})\b`), + Keywords: []string{"gsk_"}, + }, + { + // DeepSeek keys share OpenAI's `sk-<32 chars>` shape, so the + // keyword must appear within 40 chars to avoid firing on every + // sk-* literal. + ID: "deepseek-api-key", + Regex: regexp.MustCompile(`(?i:deepseek)(?:.|[\n\r]){0,40}?\b(sk-[a-z0-9]{32})\b`), + Keywords: []string{"deepseek"}, + }, + { + // AIzaSy is the Gemini-specific subtype of Google's AIza-prefixed + // keys. + ID: "google-gemini-api-key", + Regex: regexp.MustCompile(`\b(AIzaSy[A-Za-z0-9_-]{33})`), + Keywords: []string{"AIzaSy"}, + }, + { + ID: "xai-api-key", + Regex: regexp.MustCompile(`\b(xai-[0-9a-zA-Z_]{80})\b`), + Keywords: []string{"xai-"}, + }, + { + ID: "replicate-api-key", + Regex: regexp.MustCompile(`\b(r8_[0-9A-Za-z\-_]{37})\b`), + Keywords: []string{"r8_"}, + }, + { + ID: "fireworks-api-key", + Regex: regexp.MustCompile(`fw_[A-Za-z0-9]{20,44}`), + Keywords: []string{"fw_"}, + }, + // Services absent from the gitleaks v8.30.1 defaults, each anchored + // on a distinctive vendor prefix. + { + ID: "openrouter-api-key", + Regex: regexp.MustCompile(`\b(sk-or-v1-[0-9a-f]{64})\b`), + Keywords: []string{"sk-or-v1-"}, + }, + { + ID: "figma-pat", + Regex: regexp.MustCompile(`\b(figd_[A-Za-z0-9_-]{40,})\b`), + Keywords: []string{"figd_"}, + }, + { + ID: "posthog-personal-api-key", + Regex: regexp.MustCompile(`\b(phx_[A-Za-z0-9]{40,})\b`), + Keywords: []string{"phx_"}, + }, + { + ID: "cloudinary-url", + Regex: regexp.MustCompile(`cloudinary://[0-9]{15}:[A-Za-z0-9_-]{20,}@[a-zA-Z0-9_-]+`), + Keywords: []string{"cloudinary://"}, + }, + { + ID: "stripe-webhook-secret", + Regex: regexp.MustCompile(`\b(whsec_[A-Za-z0-9]{32,})\b`), + Keywords: []string{"whsec_"}, + }, + { + ID: "dockerhub-pat", + Regex: regexp.MustCompile(`\b(dckr_pat_[A-Za-z0-9_-]{20,})\b`), + Keywords: []string{"dckr_pat_"}, + }, + { + // New-format ghs_/ghu_ APPID_JWT App tokens: gitleaks' default + // rule is anchored to the legacy fixed 36-char body and never + // matches these. + ID: "github-installation-token-jwt", + Regex: regexp.MustCompile(`gh[us]_[0-9]+_[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}`), + Keywords: []string{"ghs_", "ghu_"}, + }, + { + // VixVerify GreenID apiCode. The vendor documents the value as + // client-safe (paired with a server-side verificationToken to + // start PII flows), but a committed apiCode still enables + // account enumeration and verification-status probing against + // the tenant's GreenID account -- material for AU/NZ + // financial-services customers under AUSTRAC/CDR obligations. + // Anchored on the vendor's own env-var name so unrelated + // 3-3-3-3 alnum strings (coupon codes, licence keys) don't fire. + ID: "vixverify-greenid-api-code", + Regex: regexp.MustCompile(`(?i)green[_-]?id[_-]?api[_-]?code['"]?\s*[:=]\s*['"]?([A-Za-z0-9]{3}-[A-Za-z0-9]{3}-[A-Za-z0-9]{3}-[A-Za-z0-9]{3})['"]?`), + Keywords: []string{"green_id", "greenid", "green-id", "vixverify"}, + }, + } +} diff --git a/internal/scanfile/rules.go b/internal/scanfile/rules.go new file mode 100644 index 0000000..01dc359 --- /dev/null +++ b/internal/scanfile/rules.go @@ -0,0 +1,217 @@ +package scanfile + +import ( + "math" + "regexp" + "strings" + "sync" +) + +// Rule is a single hand-written secret-detection rule. It has no relation to +// gitleaks' rule shape (see the package doc for why). There is no explicit +// secret-group field: scanfile.secretSpan takes the first capturing group +// that participated in a match, falling back to the whole match for rules +// with no group, so a rule opts into scoping the reported secret just by +// wrapping it in parentheses. +type Rule struct { + // ID is a short, stable slug. It never reaches the client directly; it + // is only used to derive Title() and to key MinEntropy overrides. + ID string + Regex *regexp.Regexp + // Keywords is a cheap case-insensitive substring prefilter: if set, the + // regex only runs on a line containing at least one keyword. + Keywords []string + // MinEntropy is the minimum Shannon entropy (bits/char) the matched text + // must have to be reported, unless it also matches + // highConfidenceSecretPatterns. Zero disables the check. + MinEntropy float64 +} + +// acronyms upper-cases well-known abbreviations when deriving Title() from +// ID; every other hyphen-separated word is simply capitalized. +var acronyms = map[string]string{ + "aws": "AWS", "api": "API", "jwt": "JWT", "iam": "IAM", "cos": "COS", + "hmac": "HMAC", "gcp": "GCP", "ibm": "IBM", "pat": "PAT", "id": "ID", + "url": "URL", "npm": "NPM", "sql": "SQL", "pem": "PEM", "sdk": "SDK", + "greenid": "GreenID", "oauth": "OAuth", "openai": "OpenAI", "xai": "xAI", + "vixverify": "VixVerify", +} + +// Title renders ID as a short, human-readable label, e.g. "aws-access-key-id" +// -> "AWS Access Key ID". It is what reaches the client, in the finding +// message and folded into the correlation fingerprint — never the raw ID or +// the regex/rule internals. +func (r Rule) Title() string { + words := strings.Split(r.ID, "-") + for i, w := range words { + if up, ok := acronyms[strings.ToLower(w)]; ok { + words[i] = up + continue + } + if w == "" { + continue + } + words[i] = strings.ToUpper(w[:1]) + w[1:] + } + return strings.Join(words, " ") +} + +var rulesOnce = sync.OnceValue(func() []Rule { + all := make([]Rule, 0, 64) + all = append(all, nullifyRules()...) + all = append(all, baselineRules()...) + return all +}) + +// rules returns the combined, compiled rule table (Nullify's own custom +// rules plus the baseline well-known-format rules), built once. +func rules() []Rule { + return rulesOnce() +} + +// lockfileNames are dependency manifests that carry long hex/base64-looking +// integrity hashes that regularly trip entropy-based heuristics. Mirrors the +// allowlist secrets/internal/scanners/gitleaks/config.go builds for the same +// reason, minus the vendored-directory half (a single-file/stdin scan only +// ever sees the file currently open in the editor, not a vendored tree). +var lockfileNames = map[string]bool{ + "package-lock.json": true, + "yarn.lock": true, + "pnpm-lock.yaml": true, + "Gemfile.lock": true, + "Cargo.lock": true, + "poetry.lock": true, + "go.sum": true, + "composer.lock": true, + "Pipfile.lock": true, +} + +func isLockfilePath(filePath string) bool { + base := filePath + if idx := strings.LastIndexAny(filePath, "/\\"); idx >= 0 { + base = filePath[idx+1:] + } + return lockfileNames[base] +} + +// testPlaceholderPatterns are test, example and placeholder values filtered +// out as false positives. Ported verbatim from +// secrets/internal/scanners/gitleaks/rules.go. +var testPlaceholderPatterns = []string{ + "AKIAIOSFODNN7EXAMPLE", + "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + "your-api-key-here", + "your_api_key_here", + "your-secret-here", + "your_secret_here", + "insert-your-key", + "insert_your_key", + "api-key-goes-here", + "api_key_goes_here", + "replace-with-your-key", + "replace_with_your_key", + "xxx", + "XXX", + "xxxxxxxx", + "XXXXXXXX", + "placeholder", + "PLACEHOLDER", + // Connection-string template idioms: the basic-auth rule captures the + // password segment of scheme://user:pass@host. + "pass", + "passwd", + "user", + "username", + "password", + "password123", + "password1234", + "test123", + "test1234", + "testtest", + "changeme", + "admin123", + "secret123", + "example", + "sample", + "dummy", + "fake", + "mock", + "null", + "undefined", + "none", + "empty", + "todo", + "fixme", +} + +var templateVarPattern = regexp.MustCompile(`^<[a-z_]+>$`) + +// highConfidenceSecretPatterns matches formats that are always real +// credentials and bypass entropy filtering; AWS access key IDs have low +// entropy (~3.2) but are never benign. The IAM unique-ID prefixes +// (AGPA/AIDA/AROA/AIPA/ANPA/ANVA) are excluded: they label non-secret +// identifiers. Ported verbatim from +// secrets/internal/scanners/gitleaks/rules.go. +var highConfidenceSecretPatterns = []*regexp.Regexp{ + regexp.MustCompile(`^(?:A3T[A-Z0-9]|AKIA|ASIA|ABIA|ACCA)[A-Z2-7]{16}$`), +} + +func isHighConfidenceSecret(secret string) bool { + for _, pattern := range highConfidenceSecretPatterns { + if pattern.MatchString(secret) { + return true + } + } + return false +} + +func isTestOrPlaceholder(secret string) bool { + lowerSecret := strings.ToLower(secret) + + for _, pattern := range testPlaceholderPatterns { + if lowerSecret == strings.ToLower(pattern) { + return true + } + } + + placeholderIndicators := []string{ + "example", "placeholder", "your-", "your_", "insert-", "insert_", + "replace-", "replace_", " 0 && !isHighConfidenceSecret(secret) && shannonEntropy(secret) < rule.MinEntropy { + continue + } + + findings = append(findings, NormalizedFinding{ + DetectorClass: detectorClassSecret, + Severity: severityHigh, + File: filePath, + StartLine: lineIdx + 1, + EndLine: lineIdx + 1, + StartColumn: startByte + 1, + EndColumn: endByte + 1, + CWE: cweHardcodedCredentials, + Message: messagePrefix + rule.Title(), + Fingerprint: fingerprint(repository, filePath, lineIdx+1, rule.Title()), + }) + + if len(findings) >= maxFindingsPerFileDefault { + return findings + } + } + } + } + + return findings +} + +// secretSpan picks the byte range to treat as "the secret" out of a +// FindAllStringSubmatchIndex match: the first capturing group that +// participated (rules that isolate the credential in a group, e.g. a regex +// anchored on surrounding punctuation or a `key = "..."` assignment), or the +// whole match when the rule has no capturing group or none of them fired for +// this particular alternation branch. This keeps entropy/placeholder checks +// — and the reported column range — scoped to the credential itself rather +// than including fixed keyword/punctuation text around it. +func secretSpan(match []int) (start, end int) { + for i := 2; i+1 < len(match); i += 2 { + if match[i] >= 0 && match[i+1] >= 0 { + return match[i], match[i+1] + } + } + return match[0], match[1] +} + +// keywordsPresent reports whether lowerLine contains any of the rule's +// keywords. A rule with no keywords always runs (it has no cheap prefilter). +func keywordsPresent(lowerLine string, keywords []string) bool { + if len(keywords) == 0 { + return true + } + for _, kw := range keywords { + if strings.Contains(lowerLine, kw) { + return true + } + } + return false +} + +// fingerprint mirrors the platform's secrets dedup key: a hash over +// repository, file, start line and a secret-type label. +func fingerprint(repository, file string, startLine int, secretType string) string { + sum := sha256.Sum256([]byte(strings.Join([]string{ + "secret", + repository, + file, + strconv.Itoa(startLine), + secretType, + }, "\x00"))) + return hex.EncodeToString(sum[:]) +} diff --git a/internal/scanfile/scanfile_test.go b/internal/scanfile/scanfile_test.go new file mode 100644 index 0000000..2f20f5b --- /dev/null +++ b/internal/scanfile/scanfile_test.go @@ -0,0 +1,133 @@ +package scanfile + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +// fakeAWSKey builds an AWS-access-key-ID-shaped fixture (AKIA + 16 chars +// from [A-Z2-7], distinct from the well-known AWS docs example key that this +// package's placeholder filter deliberately allowlists) via concatenation, +// so the 20-character string never appears contiguous in this file's own +// source text and can't trip a secret scanner on this repo's own commits. +func fakeAWSKey() string { + return "AKIA" + "ABCDEFGHIJKLMNOP" +} + +// fakeAnthropicKey builds an Anthropic-key-shaped fixture the same way. +func fakeAnthropicKey() string { + return "sk-ant-api03-" + strings.Repeat("x", 93) + "AA" +} + +func TestScanContentDetectsBaselineRule(t *testing.T) { + content := []byte("awsKey := \"" + fakeAWSKey() + "\"\n") + + findings := ScanContent(content, "config.go", "") + require.NotEmpty(t, findings) + + f := findings[0] + require.Equal(t, detectorClassSecret, f.DetectorClass) + require.Equal(t, cweHardcodedCredentials, f.CWE) + require.Equal(t, "config.go", f.File) + require.Equal(t, 1, f.StartLine) + require.Equal(t, "Hardcoded secret detected: AWS Access Key ID", f.Message) + require.NotEmpty(t, f.Fingerprint) +} + +func TestScanContentDetectsNullifyCustomRule(t *testing.T) { + key := fakeAnthropicKey() + content := []byte("const anthropicKey = \"" + key + "\"\n") + + findings := ScanContent(content, "main.go", "") + require.NotEmpty(t, findings) + require.Equal(t, "Hardcoded secret detected: Anthropic API Key", findings[0].Message) +} + +func TestScanContentCleanBufferHasNoFindings(t *testing.T) { + findings := ScanContent([]byte("package main\n\nfunc main() {}\n"), "main.go", "") + require.Empty(t, findings) +} + +func TestScanContentFiltersPlaceholderValues(t *testing.T) { + content := []byte(`apiKey = "your-api-key-here"` + "\n") + findings := ScanContent(content, "config.go", "") + require.Empty(t, findings, "placeholder values must never be reported") +} + +func TestScanContentFiltersLowEntropyGenericValues(t *testing.T) { + content := []byte(`password = "aaaaaaaaaaaaaaaa"` + "\n") + findings := ScanContent(content, "config.go", "") + require.Empty(t, findings, "low-entropy generic assignments must be filtered") +} + +func TestScanContentSkipsLockfiles(t *testing.T) { + // A value that would otherwise trip the AWS access key rule. + content := []byte("\"" + fakeAWSKey() + "\"\n") + findings := ScanContent(content, "nested/dir/package-lock.json", "") + require.Empty(t, findings, "lockfiles must be skipped entirely") +} + +func TestScanContentNeverLeaksTheSecretValue(t *testing.T) { + key := fakeAnthropicKey() + content := []byte("const anthropicKey = \"" + key + "\"\n") + + findings := ScanContent(content, "main.go", "") + require.NotEmpty(t, findings) + for _, f := range findings { + require.NotContains(t, f.Message, key) + require.NotContains(t, f.Fingerprint, key) + } +} + +// The fingerprint must stay a hash over {"secret", repository, file, +// startLine, Title} or on-save findings stop correlating across repeated +// scans of the same file. +func TestFingerprintScheme(t *testing.T) { + content := []byte("awsKey := \"" + fakeAWSKey() + "\"\n") + findings := ScanContent(content, "config.go", "owner/repo") + require.NotEmpty(t, findings) + f := findings[0] + + want := fingerprint("owner/repo", "config.go", f.StartLine, "AWS Access Key ID") + require.Equal(t, want, f.Fingerprint) +} + +func TestRuleTitleFormatsKnownAcronyms(t *testing.T) { + r := Rule{ID: "aws-access-key-id"} + require.Equal(t, "AWS Access Key ID", r.Title()) + + r = Rule{ID: "generic-api-key"} + require.Equal(t, "Generic API Key", r.Title()) +} + +func TestAllRulesCompileAndHaveUniqueIDs(t *testing.T) { + seen := map[string]bool{} + for _, r := range rules() { + require.NotNil(t, r.Regex, "rule %q must have a compiled regex", r.ID) + require.False(t, seen[r.ID], "duplicate rule id %q", r.ID) + seen[r.ID] = true + } + require.NotEmpty(t, rules()) +} + +func TestShannonEntropy(t *testing.T) { + require.InDelta(t, 0.0, shannonEntropy("aaaaaaaa"), 0.001) + require.Greater(t, shannonEntropy("f3Kq9vXz2Lm8Wp4Rb7Nt"), 3.5) +} + +func TestIsLockfilePath(t *testing.T) { + require.True(t, isLockfilePath("package-lock.json")) + require.True(t, isLockfilePath("nested/dir/go.sum")) + require.False(t, isLockfilePath("main.go")) +} + +func TestFingerprintIsDeterministic(t *testing.T) { + a := fingerprint("owner/repo", "file.go", 4, "Generic API Key") + b := fingerprint("owner/repo", "file.go", 4, "Generic API Key") + require.Equal(t, a, b) + + c := fingerprint("owner/repo", "file.go", 5, "Generic API Key") + require.NotEqual(t, a, c) +}