From 3b1ddd5cfed04016fc8056601d9ad91dd554477f Mon Sep 17 00:00:00 2001 From: b-long Date: Tue, 16 Dec 2025 14:20:37 -0500 Subject: [PATCH 1/6] add extracted-code to .gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 71d35fc..ef15d66 100644 --- a/.gitignore +++ b/.gitignore @@ -41,3 +41,6 @@ node_modules # Ignore manual test scripts manual_tests/ + +# Ignore extracted code resources +/extracted-code/ From b7dc16d647b0de9b037e6fe872762353ff623df6 Mon Sep 17 00:00:00 2001 From: b-long Date: Tue, 16 Dec 2025 14:21:53 -0500 Subject: [PATCH 2/6] annotate some of the getting-started guide --- docs/getting-started/index.mdx | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/docs/getting-started/index.mdx b/docs/getting-started/index.mdx index c0622da..b4638d7 100644 --- a/docs/getting-started/index.mdx +++ b/docs/getting-started/index.mdx @@ -20,9 +20,11 @@ The first step is to get the platform running locally. You can use the following > **Note for Apple M4 chip users:** > If you are running on an Apple M4 chip, set the Java environment variable before running any commands: +> > ```sh > export JAVA_OPTS_APPEND="-XX:UseSVE=0" > ``` +> > This resolves SIGILL with Code 134 errors when running Java processes (such as Keycloak). :::warning @@ -31,6 +33,7 @@ Not for production use.
Docker Compose + ```yaml name: opentdf volumes: @@ -314,16 +317,18 @@ services: retries: 10 ``` - +
## Update /etc/hosts In order for the services to communicate correctly you will need to update your `/etc/hosts` file. + ```shell echo -e "127.0.0.1 platform.opentdf.local\n127.0.0.1 keycloak.opentdf.local" | sudo tee -a /etc/hosts ``` + ## Trust Self Signed Certificates @@ -331,17 +336,25 @@ During the bootstrapping process `caddy` will generate self signed certificates. Example of extracting the certificate from the container. + ```shell mkdir -p ./opentdf-certs ``` + + + ```shell docker cp opentdf-caddy-1:/data/caddy/certificates/local/keycloak.opentdf.local/keycloak.opentdf.local.crt ./opentdf-certs ``` + + + ```shell docker cp opentdf-caddy-1:/data/caddy/certificates/local/platform.opentdf.local/platform.opentdf.local.crt ./opentdf-certs ``` + ### Import and Trust Certificates by Operating System From 664138df2e9abb289119d43dae2370b52d0f9592 Mon Sep 17 00:00:00 2001 From: b-long Date: Tue, 16 Dec 2025 14:22:14 -0500 Subject: [PATCH 3/6] add utility scripts to extract code --- package.json | 3 +- scripts/extract-code-resources.js | 156 ++++++++++++++++++++++++++++++ scripts/extract-code.sh | 40 ++++++++ 3 files changed, 198 insertions(+), 1 deletion(-) create mode 100644 scripts/extract-code-resources.js create mode 100755 scripts/extract-code.sh diff --git a/package.json b/package.json index 0607a80..de30e82 100644 --- a/package.json +++ b/package.json @@ -15,7 +15,8 @@ "gen-api-docs-all": "docusaurus gen-api-docs all --all-versions", "gen-api-docs-clean": "docusaurus clean-api-docs all", "check-vendored-yaml": "tsx src/openapi/check-vendored-yaml.ts", - "update-vendored-yaml": "tsx src/openapi/update-vendored-yaml.ts" + "update-vendored-yaml": "tsx src/openapi/update-vendored-yaml.ts", + "extract-code": "bash scripts/extract-code.sh" }, "dependencies": { "@docusaurus/core": "^3.6.3", diff --git a/scripts/extract-code-resources.js b/scripts/extract-code-resources.js new file mode 100644 index 0000000..8574c9b --- /dev/null +++ b/scripts/extract-code-resources.js @@ -0,0 +1,156 @@ +#!/usr/bin/env node + +/** + * Extract code resources from MDX files + * Extracts code blocks marked with comments + */ + +const fs = require('fs'); +const path = require('path'); + +// Configuration +const MDX_FILE = path.join(__dirname, '../docs/getting-started/index.mdx'); +const OUTPUT_DIR = path.join(__dirname, '../extracted-code'); + +// Resources to extract (001-006) +const RESOURCES_TO_EXTRACT = ['001', '002', '003', '004', '005', '006']; + +/** + * Parse MDX file and extract code resources + */ +function extractCodeResources() { + console.log('Reading MDX file:', MDX_FILE); + + if (!fs.existsSync(MDX_FILE)) { + console.error('Error: MDX file not found:', MDX_FILE); + process.exit(1); + } + + const content = fs.readFileSync(MDX_FILE, 'utf-8'); + const lines = content.split('\n'); + + // Create output directory + if (!fs.existsSync(OUTPUT_DIR)) { + fs.mkdirSync(OUTPUT_DIR, { recursive: true }); + console.log('Created output directory:', OUTPUT_DIR); + } + + let extractedCount = 0; + + // Process each resource + RESOURCES_TO_EXTRACT.forEach(resourceNum => { + const startMarker = ``; + const endMarker = ``; + + let extracting = false; + let codeLines = []; + let language = ''; + let inCodeBlock = false; + + for (let i = 0; i < lines.length; i++) { + const line = lines[i]; + const trimmedLine = line.trim(); + // Remove blockquote markers for comparison + const cleanLine = line.replace(/^>\s*/, ''); + + // Start extraction + if (cleanLine.includes(startMarker)) { + extracting = true; + continue; + } + + // End extraction + if (cleanLine.includes(endMarker)) { + if (extracting && codeLines.length > 0) { + // Determine filename based on language + const ext = getFileExtension(language, resourceNum); + const filename = `code-resource-${resourceNum}${ext}`; + const filepath = path.join(OUTPUT_DIR, filename); + + // Write the code to file + const code = codeLines.join('\n'); + fs.writeFileSync(filepath, code); + console.log(`✓ Extracted Resource ${resourceNum} -> ${filename} (${codeLines.length} lines)`); + extractedCount++; + } + extracting = false; + codeLines = []; + language = ''; + inCodeBlock = false; + break; + } + + // Extract code content + if (extracting) { + // Detect start of code fence (handle blockquote markers) + const codeBlockMatch = cleanLine.trim().match(/^```(\w*)/); + if (codeBlockMatch) { + if (!inCodeBlock) { + // Starting code block + language = codeBlockMatch[1] || ''; + inCodeBlock = true; + } else { + // Ending code block + inCodeBlock = false; + } + continue; + } + + // Add line if we're inside a code block (remove blockquote marker if present) + if (inCodeBlock) { + codeLines.push(cleanLine); + } + } + } + }); + + console.log(`\n✓ Successfully extracted ${extractedCount} code resources to ${OUTPUT_DIR}`); + return extractedCount; +} + +/** + * Determine file extension based on language + */ +function getFileExtension(language, resourceNum) { + // Map resource numbers to known types + const resourceTypes = { + '001': '.sh', // M4 chip export command + '002': '.yaml', // Docker Compose + '003': '.sh', // /etc/hosts update + '004': '.sh', // mkdir command + '005': '.sh', // keycloak cert extraction + '006': '.sh', // platform cert extraction + }; + + // Use resource-specific extension if available + if (resourceTypes[resourceNum]) { + return resourceTypes[resourceNum]; + } + + // Otherwise, determine from language tag + const langMap = { + 'sh': '.sh', + 'shell': '.sh', + 'bash': '.sh', + 'yaml': '.yaml', + 'yml': '.yaml', + 'json': '.json', + 'javascript': '.js', + 'js': '.js', + 'typescript': '.ts', + 'ts': '.ts', + 'python': '.py', + 'py': '.py', + 'powershell': '.ps1', + }; + + return langMap[language.toLowerCase()] || '.txt'; +} + +// Run extraction +try { + extractCodeResources(); +} catch (error) { + console.error('Error during extraction:', error.message); + process.exit(1); +} diff --git a/scripts/extract-code.sh b/scripts/extract-code.sh new file mode 100755 index 0000000..ef5203f --- /dev/null +++ b/scripts/extract-code.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +# Extract code resources from getting-started documentation +# This script extracts code blocks marked with comment markers (001-006) +# and saves them to the extracted-code directory + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" + +echo "================================================" +echo "Code Resource Extractor" +echo "================================================" +echo "" +echo "Project root: $PROJECT_ROOT" +echo "Extracting resources 001-006 from getting-started guide..." +echo "" + +# Run the Node.js extraction script +node "$SCRIPT_DIR/extract-code-resources.js" + +EXIT_CODE=$? + +if [ $EXIT_CODE -eq 0 ]; then + echo "" + echo "================================================" + echo "✓ Extraction complete!" + echo "================================================" + echo "" + echo "Extracted files are in: $PROJECT_ROOT/extracted-code" + echo "" + ls -lh "$PROJECT_ROOT/extracted-code" +else + echo "" + echo "================================================" + echo "✗ Extraction failed with exit code $EXIT_CODE" + echo "================================================" + exit $EXIT_CODE +fi From 2b2c74b30eef19d8beaa9e872e7938723d7fb029 Mon Sep 17 00:00:00 2001 From: b-long Date: Tue, 16 Dec 2025 14:29:34 -0500 Subject: [PATCH 4/6] test the getting started page --- .github/workflows/test-getting-started.yml | 199 +++++++++++++++++++++ 1 file changed, 199 insertions(+) create mode 100644 .github/workflows/test-getting-started.yml diff --git a/.github/workflows/test-getting-started.yml b/.github/workflows/test-getting-started.yml new file mode 100644 index 0000000..91b5ade --- /dev/null +++ b/.github/workflows/test-getting-started.yml @@ -0,0 +1,199 @@ +name: Test Getting Started Guide + +on: + workflow_dispatch: + pull_request: + paths: + - 'docs/getting-started/index.mdx' + - 'scripts/extract-code-resources.js' + - '.github/workflows/test-getting-started.yml' + push: + branches: + - main + paths: + - 'docs/getting-started/index.mdx' + - 'scripts/extract-code-resources.js' + - '.github/workflows/test-getting-started.yml' + +jobs: + test-getting-started: + name: Test Getting Started Documentation + runs-on: ubuntu-latest + timeout-minutes: 60 + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + cache: 'npm' + + - name: Install dependencies + run: npm ci + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: '1.25' + cache: true + + - name: Install otdfctl + run: | + go install github.com/opentdf/otdfctl@latest + echo "$HOME/go/bin" >> $GITHUB_PATH + otdfctl version || echo "otdfctl installed successfully" + + - name: Setup Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Extract code resources + run: npm run extract-code + + - name: Verify extracted files + run: | + echo "Checking extracted code resources..." + ls -lh extracted-code/ + echo "" + echo "Content of code-resource-003.sh (hosts update):" + cat extracted-code/code-resource-003.sh + echo "" + echo "First 10 lines of docker-compose.yaml:" + head -10 extracted-code/code-resource-002.yaml + + - name: Update /etc/hosts + run: | + echo "Updating /etc/hosts file..." + sudo bash extracted-code/code-resource-003.sh + echo "" + echo "Verifying /etc/hosts entries:" + grep "opentdf.local" /etc/hosts || echo "Warning: opentdf.local entries not found" + + - name: Prepare Docker Compose directory + run: | + mkdir -p opentdf-platform + cp extracted-code/code-resource-002.yaml opentdf-platform/docker-compose.yaml + echo "Docker Compose file ready:" + ls -lh opentdf-platform/ + + - name: Start OpenTDF Platform + run: | + cd opentdf-platform + echo "Starting OpenTDF platform services..." + docker-compose up -d + echo "" + echo "Waiting for services to initialize..." + sleep 30 + echo "" + echo "Docker containers:" + docker-compose ps + + - name: Wait for services to be healthy + run: | + cd opentdf-platform + echo "Waiting for services to become healthy..." + + # Wait up to 5 minutes for services to be healthy + timeout=300 + elapsed=0 + + while [ $elapsed -lt $timeout ]; do + unhealthy=$(docker-compose ps --format json | jq -r 'select(.Health != "healthy" and .Health != "") | .Name' | wc -l) + + if [ "$unhealthy" -eq 0 ]; then + echo "✓ All services are healthy!" + docker-compose ps + exit 0 + fi + + echo "Waiting for services... ($elapsed seconds elapsed)" + sleep 10 + elapsed=$((elapsed + 10)) + done + + echo "⚠ Timeout waiting for services to become healthy" + docker-compose ps + docker-compose logs --tail=50 + exit 1 + + - name: Create certificate directory + run: | + echo "Creating certificate directory..." + bash extracted-code/code-resource-004.sh + ls -lh opentdf-certs/ || echo "Directory created" + + - name: Extract Keycloak certificate + run: | + echo "Extracting Keycloak certificate..." + cd opentdf-platform + bash ../extracted-code/code-resource-005.sh + echo "" + echo "Certificate extracted:" + ls -lh ../opentdf-certs/keycloak.opentdf.local.crt || echo "Certificate not found" + + - name: Extract Platform certificate + run: | + echo "Extracting Platform certificate..." + cd opentdf-platform + bash ../extracted-code/code-resource-006.sh + echo "" + echo "Certificates extracted:" + ls -lh ../opentdf-certs/ + + - name: Verify certificates + run: | + echo "Verifying certificate files..." + if [ -f "opentdf-certs/keycloak.opentdf.local.crt" ]; then + echo "✓ Keycloak certificate found" + openssl x509 -in opentdf-certs/keycloak.opentdf.local.crt -noout -subject -dates + else + echo "✗ Keycloak certificate missing" + exit 1 + fi + + echo "" + + if [ -f "opentdf-certs/platform.opentdf.local.crt" ]; then + echo "✓ Platform certificate found" + openssl x509 -in opentdf-certs/platform.opentdf.local.crt -noout -subject -dates + else + echo "✗ Platform certificate missing" + exit 1 + fi + + - name: Test platform connectivity + run: | + echo "Testing platform connectivity..." + + # Test Keycloak + echo "Testing Keycloak (https://keycloak.opentdf.local:9443)..." + curl -k -f -s -o /dev/null -w "%{http_code}\n" https://keycloak.opentdf.local:9443 || echo "Keycloak not responding" + + # Test Platform + echo "Testing Platform (https://platform.opentdf.local:8443)..." + curl -k -f -s -o /dev/null -w "%{http_code}\n" https://platform.opentdf.local:8443 || echo "Platform not responding" + + - name: Collect logs on failure + if: failure() + run: | + cd opentdf-platform + echo "=== Docker Compose Services ===" + docker-compose ps + echo "" + echo "=== Docker Compose Logs ===" + docker-compose logs --tail=100 + echo "" + echo "=== Docker System Info ===" + docker system df + docker images + + - name: Cleanup + if: always() + run: | + cd opentdf-platform || exit 0 + echo "Stopping and removing containers..." + docker-compose down -v || true + cd .. + rm -rf opentdf-platform opentdf-certs extracted-code || true From c52d1601c73d648b8e561c3d7c4f25a8b5f70f8d Mon Sep 17 00:00:00 2001 From: b-long Date: Tue, 16 Dec 2025 14:31:36 -0500 Subject: [PATCH 5/6] remove duplicates --- docs/getting-started/index.mdx | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/getting-started/index.mdx b/docs/getting-started/index.mdx index b4638d7..c310295 100644 --- a/docs/getting-started/index.mdx +++ b/docs/getting-started/index.mdx @@ -341,14 +341,12 @@ Example of extracting the certificate from the container. mkdir -p ./opentdf-certs ``` - ```shell docker cp opentdf-caddy-1:/data/caddy/certificates/local/keycloak.opentdf.local/keycloak.opentdf.local.crt ./opentdf-certs ``` - ```shell From 60b6badd36ac6ec4baa2609b6b25ac6d0c03d1b0 Mon Sep 17 00:00:00 2001 From: b-long Date: Tue, 16 Dec 2025 14:34:54 -0500 Subject: [PATCH 6/6] test the getting started page --- .github/workflows/test-getting-started.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test-getting-started.yml b/.github/workflows/test-getting-started.yml index 91b5ade..8b23bdf 100644 --- a/.github/workflows/test-getting-started.yml +++ b/.github/workflows/test-getting-started.yml @@ -82,13 +82,13 @@ jobs: run: | cd opentdf-platform echo "Starting OpenTDF platform services..." - docker-compose up -d + docker compose up -d echo "" echo "Waiting for services to initialize..." sleep 30 echo "" echo "Docker containers:" - docker-compose ps + docker compose ps - name: Wait for services to be healthy run: | @@ -100,11 +100,11 @@ jobs: elapsed=0 while [ $elapsed -lt $timeout ]; do - unhealthy=$(docker-compose ps --format json | jq -r 'select(.Health != "healthy" and .Health != "") | .Name' | wc -l) + unhealthy=$(docker compose ps --format json | jq -r 'select(.Health != "healthy" and .Health != "") | .Name' | wc -l) if [ "$unhealthy" -eq 0 ]; then echo "✓ All services are healthy!" - docker-compose ps + docker compose ps exit 0 fi @@ -114,8 +114,8 @@ jobs: done echo "⚠ Timeout waiting for services to become healthy" - docker-compose ps - docker-compose logs --tail=50 + docker compose ps + docker compose logs --tail=50 exit 1 - name: Create certificate directory @@ -180,10 +180,10 @@ jobs: run: | cd opentdf-platform echo "=== Docker Compose Services ===" - docker-compose ps + docker compose ps echo "" echo "=== Docker Compose Logs ===" - docker-compose logs --tail=100 + docker compose logs --tail=100 echo "" echo "=== Docker System Info ===" docker system df @@ -194,6 +194,6 @@ jobs: run: | cd opentdf-platform || exit 0 echo "Stopping and removing containers..." - docker-compose down -v || true + docker compose down -v || true cd .. rm -rf opentdf-platform opentdf-certs extracted-code || true