dot-context-pack-build.sh Phase 4a P1+P2 snapshot (Stage A+B+C)
#!/usr/bin/env bash
=============================================================================
dot-context-pack-build — Context Pack Builder (Đ43 "động cơ chính")
=============================================================================
@version 0.1-skeleton
@date 2026-04-17
@author Claude CLI (claude-go), S178 Fix 12 Phase 4a P1+P2
@spec Điều 43 v1.2 FINAL rev 3 §6 (8 bước) + §5 (schema) + §6.X CẤM HARDCODE
@paired-dot dot-context-pack-verify.sh (NT12 HP v4.6.2 — sẽ soạn Phase 4b)
@db-access directus (KHO): R/W context_pack_*, R dot_tools, normative_registry,
birth_registry, meta_catalog, dot_operations, dot_config
incomex_metadata (NÃO): R kb_documents (Phase 4a P4 phiên sau)
postgres catalog: pg_database (db_count khi whitelist rỗng)
@gateway psql (Unix socket trust in container) — local sync, no network
Role directus (owner directus DB) cho write
Role context_pack_readonly cho read cross-DB (Phase 1.5)
@exit-codes 0 OK / unchanged-skip / coalesced-skip
1 ERROR (runtime / validate / publish)
2 USAGE
3 PRECHECK_FAIL (PG / KB / infra)
4 LOCK_COALESCED (reserve, current maps to 0)
@scope Phase 4a P1+P2 = skeleton + Bước 1-4. Bước 5-8 Phase 4a P3+P4 phiên sau.
@bootstrap Đ33 §13 ngoại lệ — chạy manual trong dev, register dot_tools ở Phase 5.
=============================================================================
BƯỚC MAP (§6 Đ43):
Bước 1 precheck() — PG/KB health, git_commit 5-tier, on-deploy gate
Bước 2 try_lock() — pg_try_advisory_lock(43,1), coalesce-skip, INSERT request
Bước 3 query_pg() — law/dot/entity/species/db counts (reference tables + dot_config)
Bước 4 scan_fs() — loop dot_config.context_pack_scan_paths, mtime cache
Bước 5 generate() — render 8 section + 2 checksum (P3 phiên sau)
Bước 6 validate() — min/max size + format + compare-unchanged (P3 phiên sau)
Bước 7 publish() — 7a..7g 2-phase publish + repair (P4 phiên sau)
Bước 8 release() — pg_advisory_unlock, UPDATE request done (P4 phiên sau)
CẤM HARDCODE (§6.X): section list, folder, DB, pattern, threshold, path, size.
Mọi giá trị qua dot_config hoặc reference table PG hoặc catalog pg_*.
=============================================================================
set -euo pipefail
-----------------------------------------------------------------------------
Globals (readonly constants — không phải config runtime)
-----------------------------------------------------------------------------
readonly VERSION="0.1-skeleton" SCRIPT_NAME="$(basename "$0")" readonly SCRIPT_NAME readonly PG_CONTAINER="${PG_CONTAINER:-postgres}" readonly PG_USER_RW="${PG_USER_RW:-directus}" readonly PG_DB_MAIN="${PG_DB_MAIN:-directus}" readonly PG_USER_RO="${PG_USER_RO:-context_pack_readonly}" readonly PG_DB_NAO="${PG_DB_NAO:-incomex_metadata}" readonly LOCK_NS_BUILD_CLASSID=43 readonly LOCK_NS_BUILD_OBJID=1 readonly TMPDIR="${TMPDIR:-/tmp}"
-----------------------------------------------------------------------------
Runtime state (populated at parse_args + runtime)
-----------------------------------------------------------------------------
DRY_RUN=0 REPAIR=0 VERBOSE=0 TRIGGER_SOURCE=""
shellcheck disable=SC2034 # BUILD_ID set Stage B in try_lock, used Stage B/C/P3/P4
BUILD_ID=""
shellcheck disable=SC2034 # REQUEST_ID set Stage B in try_lock, used Stage C/P4/P5
REQUEST_ID="" LOCK_HELD=0
shellcheck disable=SC2034 # OUTPUT_ROOT resolved Stage B from dot_config, used Stage B/C/P3/P4
OUTPUT_ROOT="" GIT_COMMIT="" DEDUPE_BUCKET="" LAW_COUNT=0 DOT_COUNT=0 ENTITY_COUNT=0 SPECIES_COUNT=0 DB_COUNT=0 SCANNED_FILE_COUNT=0
-----------------------------------------------------------------------------
Logging — stdout INFO/OK/SKIP/DRY, stderr WARN/ERR/FATAL
-----------------------------------------------------------------------------
log_info() { printf '[INFO] %s\n' "$"; } log_ok() { printf '[OK] %s\n' "$"; } log_skip() { printf '[SKIP] %s\n' "$"; } log_dry() { printf '[DRY] %s\n' "$"; } log_debug() { if [[ $VERBOSE -eq 1 ]]; then printf '[DEBUG] %s\n' "$"; fi; } log_warn() { printf '[WARN] %s\n' "$" >&2; } log_err() { printf '[ERR] %s\n' "$" >&2; } log_fatal() { printf '[FATAL] %s\n' "$" >&2; }
Stub for fn_log_issue (Đ35 v5.1 BLOCK 4 chưa enact — TD-S178-12 retrofit sau)
log_issue() { local severity="${1:-warn}" local category="${2:-generic}" local summary="${3:-}" log_warn "log_issue STUB: severity=${severity} category=${category} summary="${summary}"" return 0 }
-----------------------------------------------------------------------------
Cleanup trap — release advisory lock best-effort nếu hold
-----------------------------------------------------------------------------
on_exit() {
local rc=$?
if [[ $LOCK_HELD -eq 1 ]]; then
log_debug "trap EXIT: releasing advisory lock (${LOCK_NS_BUILD_CLASSID},${LOCK_NS_BUILD_OBJID})"
docker exec -i "$PG_CONTAINER" psql -U "$PG_USER_RW" -d "$PG_DB_MAIN" -At
-c "SELECT pg_advisory_unlock(${LOCK_NS_BUILD_CLASSID}, ${LOCK_NS_BUILD_OBJID});"
>/dev/null 2>&1 || true
LOCK_HELD=0
fi
return $rc
}
trap on_exit EXIT
on_err() { local line="${1:-?}" log_fatal "Error at line ${line} (exit $?)" exit 1 } trap 'on_err $LINENO' ERR
-----------------------------------------------------------------------------
PG helpers — stubs (Stage B implement đầy đủ)
-----------------------------------------------------------------------------
run_pg_rw() { docker exec -i "$PG_CONTAINER" psql -U "$PG_USER_RW" -d "$PG_DB_MAIN" -At -v ON_ERROR_STOP=1 <<< "$1" }
run_pg_ro_db() { local db="$1" local sql="$2" docker exec -i "$PG_CONTAINER" psql -U "$PG_USER_RO" -d "$db" -At -v ON_ERROR_STOP=1 <<< "$sql" }
table_exists_in_db() { local db="$1" local tbl="$2" local result result="$(run_pg_ro_db "$db" "SELECT 1 FROM information_schema.tables WHERE table_schema='public' AND table_name='${tbl}' LIMIT 1" 2>/dev/null || true)" [[ "$result" == "1" ]] }
-----------------------------------------------------------------------------
Config loader — CẤM fallback hardcode (§6.X P2 CẤM fallback khi JSONB thiếu key)
-----------------------------------------------------------------------------
dot_config_get() { local key="$1" local value value="$(run_pg_rw "SELECT value FROM dot_config WHERE key='${key}'")" if [[ -z "$value" ]]; then log_fatal "dot_config key missing: ${key} (§6.X P2 CẤM fallback)" exit 1 fi printf '%s' "$value" }
-----------------------------------------------------------------------------
KB env loader (Agent Data API credentials)
-----------------------------------------------------------------------------
kb_env_load() {
TD-S178-14: Đ33 §14 spec /opt/incomex/secrets/.env.production không tồn tại trên VPS hiện tại.
Fallback dùng /opt/incomex/docker/.env (đã chứa AGENT_DATA_* từ Phase 1.5 verify).
local kb_env="${KB_ENV_FILE:-/opt/incomex/docker/.env}" if [[ ! -r "$kb_env" ]]; then log_fatal "KB env file not readable: $kb_env" exit 3 fi set -a
shellcheck source=/dev/null
source "$kb_env" set +a if [[ -z "${AGENT_DATA_URL:-}" || -z "${AGENT_DATA_API_KEY:-}" ]]; then log_fatal "KB env missing AGENT_DATA_URL or AGENT_DATA_API_KEY in $kb_env" exit 3 fi }
-----------------------------------------------------------------------------
git_commit 5-tier fallback (§6 Bước 1 giữ v1.1)
-----------------------------------------------------------------------------
detect_git_commit() {
Tier 1: git rev-parse in /opt/incomex/*-repo/
local repo sha for repo in /opt/incomex/*-repo; do [[ -d "$repo/.git" ]] || continue sha="$(git -C "$repo" rev-parse HEAD 2>/dev/null || true)" if [[ -n "$sha" ]]; then GIT_COMMIT="$sha" log_debug "git_commit tier1 from $repo" return 0 fi done
Tier 2: /opt/incomex/RELEASE_VERSION
if [[ -r /opt/incomex/RELEASE_VERSION ]]; then GIT_COMMIT="$(tr -d '[:space:]' < /opt/incomex/RELEASE_VERSION)" if [[ -n "$GIT_COMMIT" ]]; then log_debug "git_commit tier2 from RELEASE_VERSION" return 0 fi fi
Tier 3: env RELEASE_SHA
if [[ -n "${RELEASE_SHA:-}" ]]; then GIT_COMMIT="$RELEASE_SHA" log_debug "git_commit tier3 from $RELEASE_SHA" return 0 fi
Tier 4: vps_deploy_log latest (nếu bảng tồn tại)
if table_exists_in_db "$PG_DB_MAIN" "vps_deploy_log"; then sha="$(run_pg_rw "SELECT git_sha FROM vps_deploy_log ORDER BY deployed_at DESC LIMIT 1" 2>/dev/null || true)" if [[ -n "$sha" ]]; then GIT_COMMIT="$sha" log_debug "git_commit tier4 from vps_deploy_log" return 0 fi fi
Tier 5: unknown + WARN
GIT_COMMIT="unknown" log_warn "git_commit=unknown (tier 5, all fallbacks exhausted — TD-S178-17 vps_deploy_log missing)" }
-----------------------------------------------------------------------------
Đ41 §6.5 — on-deploy gate (chỉ chạy khi is_known_good=true)
-----------------------------------------------------------------------------
on_deploy_gate() { [[ "$TRIGGER_SOURCE" != "on_deploy" ]] && return 0 if ! table_exists_in_db "$PG_DB_MAIN" "vps_deploy_log"; then log_warn "on_deploy requested nhưng vps_deploy_log missing (TD-S178-17) — permissive skip gate, tiếp build" return 0 fi local known_good known_good="$(run_pg_rw "SELECT is_known_good FROM vps_deploy_log ORDER BY deployed_at DESC LIMIT 1" 2>/dev/null || echo 'f')" if [[ "$known_good" != "t" ]]; then log_skip "on_deploy: latest deploy is_known_good=${known_good} — exit 0 per Đ41 §6.5" exit 0 fi log_ok "on_deploy gate PASS (is_known_good=true)" }
-----------------------------------------------------------------------------
Usage
-----------------------------------------------------------------------------
usage() { cat <<USAGE Usage: ${SCRIPT_NAME} [OPTIONS]
Đ43 v1.2 rev 3 §6 — Context pack builder (8 bước).
OPTIONS: --help, -h In hướng dẫn này --dry-run Chạy 8 bước KHÔNG ghi PG live / KB / FS live --trigger-source=<code> Nguồn trigger (§5.1 Đ43, 6 enum values): cron, on_demand, on_deploy, on_law_enact, on_dot_register, system_init Default: on_demand --repair Mode repair §6 Bước 7g (phát hiện state post_fs_pre_db_finalize → finalize hoặc rollback) --build-id=<id> Force build_id (debug / repair); default auto-generate --verbose Debug log
EXIT CODES: 0 OK (kể cả coalesced-skip / unchanged-skip) 1 ERROR 2 USAGE 3 PRECHECK_FAIL USAGE }
-----------------------------------------------------------------------------
Argument parser
-----------------------------------------------------------------------------
shellcheck disable=SC2034 # BUILD_ID / REQUEST_ID / OUTPUT_ROOT used in Stage B+C (currently stub)
parse_args() { while [[ $# -gt 0 ]]; do case "$1" in --help|-h) usage; exit 0 ;; --dry-run) DRY_RUN=1; shift ;; --repair) REPAIR=1; shift ;; --verbose) VERBOSE=1; shift ;; --trigger-source=) TRIGGER_SOURCE="${1#=}"; shift ;; --trigger-source) [[ $# -lt 2 ]] && { log_err "--trigger-source requires value"; exit 2; } TRIGGER_SOURCE="$2"; shift 2 ;; --build-id=) BUILD_ID="${1#=}"; shift ;; --build-id) [[ $# -lt 2 ]] && { log_err "--build-id requires value"; exit 2; } BUILD_ID="$2"; shift 2 ;; *) log_err "Unknown option: $1" usage >&2 exit 2 ;; esac done
TRIGGER_SOURCE="${TRIGGER_SOURCE:-on_demand}"
case "$TRIGGER_SOURCE" in cron|on_demand|on_deploy|on_law_enact|on_dot_register|system_init) ;; *) log_err "Invalid --trigger-source: ${TRIGGER_SOURCE} (valid: cron|on_demand|on_deploy|on_law_enact|on_dot_register|system_init per §5.1)" exit 2 ;; esac }
=============================================================================
8 BƯỚC §6 — stubs (implementation tuần tự: Stage B = 1+2, Stage C = 3+4, P3+P4 = 5-8)
=============================================================================
-----------------------------------------------------------------------------
Bước 1 §6 — PRECHECK
-----------------------------------------------------------------------------
precheck() {
1.1 PG health (directus DB)
if ! run_pg_rw "SELECT 1" >/dev/null 2>&1; then log_fatal "PG health fail (container=${PG_CONTAINER}, user=${PG_USER_RW}, db=${PG_DB_MAIN})" exit 3 fi log_ok "PG healthy (${PG_DB_MAIN})"
1.2 OUTPUT_ROOT từ dot_config (CẤM hardcode §6.X)
OUTPUT_ROOT="$(dot_config_get 'context_pack_output_root')" log_info "OUTPUT_ROOT=${OUTPUT_ROOT} (from dot_config)"
1.3 Folder existence check (Phase 2 đã tạo 3 folders)
local dir for dir in "$OUTPUT_ROOT" "${OUTPUT_ROOT}.tmp" "${OUTPUT_ROOT}-staging"; do if [[ ! -d "$dir" ]]; then log_fatal "required folder missing: ${dir} (rerun dot-dieu43-fs-init.sh Phase 2)" exit 3 fi done log_ok "output folders exist: ${OUTPUT_ROOT}{,.tmp,-staging}"
1.4 Lock dir writable check (use tmp folder as lock dir proxy)
if [[ ! -w "${OUTPUT_ROOT}.tmp" ]]; then log_warn "lock dir not writable as $(whoami): ${OUTPUT_ROOT}.tmp (need docker/sudo for write in Bước 5)" else log_ok "lock dir writable: ${OUTPUT_ROOT}.tmp" fi
1.5 dot_operations có CONTEXT_PACK_BUILD (Phase 1 migration đã seed §5.6)
local op_count op_count="$(run_pg_rw "SELECT COUNT(*) FROM dot_operations WHERE code='CONTEXT_PACK_BUILD'")" if [[ "$op_count" != "1" ]]; then log_fatal "dot_operations missing CONTEXT_PACK_BUILD (count=${op_count}, expect 1) — Phase 1 migration incomplete" exit 3 fi log_ok "dot_operations.CONTEXT_PACK_BUILD present"
1.6 KB API health
kb_env_load
local http_code
http_code="$(curl -sS -o /dev/null -w '%{http_code}' --max-time 10
-H "X-API-Key: ${AGENT_DATA_API_KEY}"
"${AGENT_DATA_URL}/health" 2>/dev/null || echo '000')"
if [[ "$http_code" != "200" ]]; then
log_fatal "KB API health fail HTTP=${http_code} (URL=${AGENT_DATA_URL})"
exit 3
fi
log_ok "KB API healthy (${AGENT_DATA_URL})"
1.7 git_commit 5-tier fallback
detect_git_commit log_info "git_commit=${GIT_COMMIT}"
1.8 on-deploy gate Đ41 §6.5
on_deploy_gate }
-----------------------------------------------------------------------------
Bước 2 §6 — TRY-LOCK + coalesce-skip + INSERT request
-----------------------------------------------------------------------------
try_lock() {
2.1 Compute dedupe_bucket (hourly, per trigger)
DEDUPE_BUCKET="$(run_pg_rw "SELECT date_trunc('hour', now())::text")" log_debug "dedupe_bucket=${DEDUPE_BUCKET}"
2.2 try acquire advisory lock (43, 1) — namespace build per §6 v1.1
local got got="$(run_pg_rw "SELECT pg_try_advisory_lock(${LOCK_NS_BUILD_CLASSID}, ${LOCK_NS_BUILD_OBJID})")"
if [[ "$got" != "t" ]]; then # 2.3a Busy → coalesce skip (§6 Bước 2 v1.1) log_skip "advisory_lock(${LOCK_NS_BUILD_CLASSID},${LOCK_NS_BUILD_OBJID}) BUSY — coalesce skip" if [[ $DRY_RUN -eq 1 ]]; then log_dry "would INSERT request (trigger=${TRIGGER_SOURCE}, bucket=${DEDUPE_BUCKET}, status=skipped, reason=coalesced)" else run_pg_rw "INSERT INTO context_pack_requests (trigger_source, dedupe_bucket, status, detail) VALUES ('${TRIGGER_SOURCE}', '${DEDUPE_BUCKET}', 'skipped', '{"reason":"coalesced"}'::jsonb)" >/dev/null || true log_ok "coalesce-skip request inserted" fi exit 0 fi
2.3b Lock acquired
LOCK_HELD=1 log_ok "advisory_lock(${LOCK_NS_BUILD_CLASSID},${LOCK_NS_BUILD_OBJID}) acquired"
2.4 Generate build_id nếu --build-id không set
if [[ -z "$BUILD_ID" ]]; then # Avoid tr|head SIGPIPE (141) under pipefail; use bash $RANDOM local _rand printf -v _rand '%04x%02x' "$RANDOM" "$((RANDOM % 256))" BUILD_ID="$(date -u +%Y%m%d-%H%M%S)-${_rand}" fi log_info "build_id=${BUILD_ID}"
2.5 INSERT request status='running'
if [[ $DRY_RUN -eq 1 ]]; then log_dry "would INSERT request (trigger=${TRIGGER_SOURCE}, bucket=${DEDUPE_BUCKET}, status=running, build_id=${BUILD_ID})" REQUEST_ID="dry-run" else REQUEST_ID="$(run_pg_rw "INSERT INTO context_pack_requests (trigger_source, dedupe_bucket, status, started_at, detail) VALUES ('${TRIGGER_SOURCE}', '${DEDUPE_BUCKET}', 'running', now(), jsonb_build_object('build_id', '${BUILD_ID}')) RETURNING id" | head -1)" log_ok "request_id=${REQUEST_ID} inserted (status=running)" fi }
-----------------------------------------------------------------------------
Bước 3 §6 — QUERY PG (reference tables + dot_config whitelist + pg_database fallback)
Sources per prompt §3 Stage C + §6 Đ43 v1.2 rev 3:
law_count ← normative_registry
dot_count ← dot_tools
entity_count ← birth_registry
species_count ← meta_catalog
db_count ← dot_config.context_pack_scan_db_whitelist
rỗng → pg_database catalog exclude (postgres,template0,template1) [NT11]
-----------------------------------------------------------------------------
query_pg() { local db_whitelist_json whitelist_len db_whitelist_json="$(dot_config_get 'context_pack_scan_db_whitelist')" whitelist_len="$(jq 'length' <<< "$db_whitelist_json")"
if [[ "$whitelist_len" == "0" ]]; then # NT11: khai tối thiểu — pg_database catalog thay hardcode số DB DB_COUNT="$(run_pg_ro_db "$PG_DB_MAIN" "SELECT COUNT(*) FROM pg_database WHERE datname NOT IN ('template0','template1','postgres')")" log_debug "db_count: pg_database catalog (NT11, whitelist empty) → ${DB_COUNT}" else DB_COUNT="$whitelist_len" log_debug "db_count: context_pack_scan_db_whitelist length → ${DB_COUNT}" fi
Counts from authoritative reference tables (read-only role cross-DB Phase 1.5 P10)
LAW_COUNT="$(run_pg_ro_db "$PG_DB_MAIN" "SELECT COUNT() FROM normative_registry")" DOT_COUNT="$(run_pg_ro_db "$PG_DB_MAIN" "SELECT COUNT() FROM dot_tools")" ENTITY_COUNT="$(run_pg_ro_db "$PG_DB_MAIN" "SELECT COUNT() FROM birth_registry")" SPECIES_COUNT="$(run_pg_ro_db "$PG_DB_MAIN" "SELECT COUNT() FROM meta_catalog")"
log_ok "query_pg: law=${LAW_COUNT} dot=${DOT_COUNT} entity=${ENTITY_COUNT} species=${SPECIES_COUNT} db=${DB_COUNT}" }
-----------------------------------------------------------------------------
Bước 4 §6 — SCAN FS (dot_config.context_pack_scan_paths, mtime cache)
Cache: TSV per build — path\ttotal_files\tlatest_mtime_epoch
-----------------------------------------------------------------------------
scan_fs() { local paths_json paths_json="$(dot_config_get 'context_pack_scan_paths')"
local tmp_cache="${TMPDIR}/dcp-scan-${BUILD_ID:-nobuild}.tsv" : > "$tmp_cache"
local total=0 skipped=0 local path count newest while IFS= read -r path; do [[ -z "$path" ]] && continue if [[ ! -d "$path" ]]; then log_warn "scan_fs: folder missing → skip: ${path}" skipped=$((skipped + 1)) continue fi count="$(find "$path" -maxdepth 3 -type f 2>/dev/null | wc -l)" # awk avoids SIGPIPE under pipefail (processes all input, emits once) newest="$(find "$path" -maxdepth 3 -type f -printf '%T@\n' 2>/dev/null | awk '$1 > max {max=$1} END {print max+0}')" printf '%s\t%s\t%s\n' "$path" "$count" "$newest" >> "$tmp_cache" total=$((total + count)) log_debug "scan_fs: ${path} total=${count} newest_mtime=${newest}" done < <(jq -r '.[]' <<< "$paths_json")
SCANNED_FILE_COUNT="$total" log_ok "scan_fs: scanned=${SCANNED_FILE_COUNT} skipped=${skipped} cache=${tmp_cache}" }
Bước 5 §6 Đ43 — P3 phiên sau
generate() { log_info "Bước 5 generate stub (P3 phiên sau)" return 0 }
Bước 6 §6 Đ43 — P3 phiên sau
validate() { log_info "Bước 6 validate stub (P3 phiên sau)" return 0 }
Bước 7 §6 Đ43 — P4 phiên sau
publish() { log_info "Bước 7 publish stub (P4 phiên sau)" return 0 }
Bước 8 §6 Đ43 — P4 phiên sau
release() { log_info "Bước 8 release stub (P4 phiên sau)" return 0 }
Repair mode alternative flow — §6 Bước 7g (Phase 4a P4 implement)
repair_publish() { log_info "repair_publish stub (P4 phiên sau)" return 0 }
=============================================================================
Main flow
=============================================================================
main() { parse_args "$@"
log_info "${SCRIPT_NAME} v${VERSION}" log_info "trigger_source=${TRIGGER_SOURCE} dry_run=${DRY_RUN} repair=${REPAIR} verbose=${VERBOSE}"
if [[ $REPAIR -eq 1 ]]; then log_info "=== REPAIR MODE (§6 Bước 7g) ===" repair_publish log_ok "${SCRIPT_NAME} (repair) completed" exit 0 fi
log_info "=== Bước 1 PRECHECK ===" precheck
log_info "=== Bước 2 TRY-LOCK ===" try_lock
log_info "=== Bước 3 QUERY PG ===" query_pg
log_info "=== Bước 4 SCAN FS ===" scan_fs
log_info "=== Bước 5 GENERATE + 2 CHECKSUM ===" generate
log_info "=== Bước 6 VALIDATE ===" validate
log_info "=== Bước 7 PUBLISH ===" publish
log_info "=== Bước 8 RELEASE ===" release
log_ok "${SCRIPT_NAME} completed (trigger=${TRIGGER_SOURCE} dry_run=${DRY_RUN})" exit 0 }
main "$@"