Mullassery/PyReverseETL

README Quick start: create-workflow (postgres customers) and create-activation (webhook), then `pyreverseetl execute ltv_to_hook --dry-run` (README Dry run section). A full execute to a live webhook was also run once and delivered 2 rows.

cd <host-path> && ls -la && cat README* 2>/dev/null | head -300
cd <host-path> && ls -la .github/workflows 2>/dev/null; cat pyproject.toml setup.py 2>/dev/null | head -80; ls <host-path>; which python3 node npm uv pip3 2>&1
cd <host-path>; python3 --version; which cargo rustc docker pip pip3 uv maturin psql 2>&1; sed -n 1,200p <host-path> | head -150; cat rust-toolchain.toml
export DEBIAN_FRONTEND=noninteractive; (sudo apt-get update -qq && sudo apt-get install -y -qq python3-venv python3-pip build-essential pkg-config libssl-dev curl ca-certificates) > /tmp/apt.log 2>&1; echo apt=$?; tail -3 /tmp/apt.log
curl -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain 1.97 > /tmp/rustup.log 2>&1; echo rustup=$?; tail -3 /tmp/rustup.log
cd <host-path> && source $HOME/.cargo/env && rustc --version && python3 -m venv <host-path> && <host-path> install -q maturin pytest 2>&1 | tail -3; export NODE_USE_ENV_PROXY=1; <host-path> --version && (timeout 280 <host-path> build --release -o /tmp/wheels > /tmp/build.log 2>&1; echo build=$?); tail -5 /tmp/build.log
grep -nE "error|panicked|failed to|Compiling" /tmp/build.log | grep -v "Compiling" | head -30; grep -c Compiling /tmp/build.log
sed -n 860,890p /tmp/build.log | cut -c1-300
cd <host-path> && source $HOME/.cargo/env && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y -qq libsasl2-dev >/tmp/apt2.log 2>&1; echo apt=$?; (timeout 280 <host-path> build --release -o /tmp/wheels > /tmp/build.log 2>&1; echo build=$?); grep -nE "^error|panicked" /tmp/build.log | head; tail -2 /tmp/build.log | cut -c1-300
cd <host-path> && source $HOME/.cargo/env && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y -qq cmake zlib1g-dev libcurl4-openssl-dev >/tmp/apt3.log 2>&1; echo apt=$?; (timeout 290 <host-path> build --release -o /tmp/wheels > /tmp/build.log 2>&1; echo build=$?); grep -nE "^error|panicked" /tmp/build.log | head -5; tail -2 /tmp/build.log | cut -c1-300
sed -n 1,55p /tmp/build.log | grep -vE "^\s*(Compiling|Downloaded|Checking)" | cut -c1-250 | tail -40
grep -n -B2 -A12 "aws-sdk-s3" /tmp/build.log | grep -E "error|warning: unused|-->|rustc|cargo" | cut -c1-250 | head -30
sed -n 57,61p /tmp/build.log | cut -c1-1500 | tail -c 1500
grep -n -iE "killed|signal|out of memory|SIGKILL|rustc-LLVM|LLVM ERROR|error\[" /tmp/build.log | cut -c1-300 | head; free -m; nproc; dmesg 2>/dev/null | tail -3
cd <host-path> && source $HOME/.cargo/env && export CARGO_PROFILE_RELEASE_LTO=false CARGO_PROFILE_RELEASE_CODEGEN_UNITS=16 CARGO_BUILD_JOBS=2 && nohup sh -c 'cd <host-path> && cargo build --release --lib > /tmp/cargo.log 2>&1; echo EXIT=$? >> /tmp/cargo.log' >/dev/null 2>&1 &
sleep 5; tail -2 /tmp/cargo.log
sleep 240; tail -3 /tmp/cargo.log | cut -c1-300; free -m | head -2
sleep 280; tail -3 /tmp/cargo.log | cut -c1-300
cd <host-path> && source $HOME/.cargo/env && export CARGO_PROFILE_RELEASE_LTO=false CARGO_PROFILE_RELEASE_CODEGEN_UNITS=16 CARGO_BUILD_JOBS=2 && (timeout 280 <host-path> build --release -o /tmp/wheels > /tmp/build2.log 2>&1; echo build=$?); tail -2 /tmp/build2.log | cut -c1-200; ls /tmp/wheels && <host-path> install -q /tmp/wheels/*.whl 2>&1 | tail -2; <host-path> --version; echo rc=$?
cd <host-path> && source $HOME/.cargo/env && export CARGO_PROFILE_RELEASE_LTO=false CARGO_PROFILE_RELEASE_CODEGEN_UNITS=16 CARGO_BUILD_JOBS=2 && <host-path> install -q patchelf && rm -rf /tmp/wheels && (timeout 280 <host-path> build --release -o /tmp/wheels > /tmp/build2.log 2>&1; echo build=$?); tail -1 /tmp/build2.log | cut -c1-200; <host-path> install -q --force-reinstall /tmp/wheels/*.whl 2>&1 | tail -2; <host-path> --version; echo rc=$?
grep -n "Caused\|Error\|error" /tmp/build2.log | cut -c1-300 | head; ls <host-path> | head -30
cd <host-path> && source $HOME/.cargo/env && export PATH=<host-path> CARGO_PROFILE_RELEASE_LTO=false CARGO_PROFILE_RELEASE_CODEGEN_UNITS=16 CARGO_BUILD_JOBS=2 && rm -rf /tmp/wheels && (timeout 280 maturin build --release -o /tmp/wheels > /tmp/build2.log 2>&1; echo build=$?); tail -1 /tmp/build2.log | cut -c1-200; pip install -q --force-reinstall /tmp/wheels/*.whl 2>&1 | tail -2; pyreverseetl --version; echo rc=$?
source <host-path>; pyreverseetl --help 2>&1 | head -40; which docker psql 2>&1; sudo -n true && echo sudo_ok
cd <host-path> && source <host-path>; grep -n "elif cmd\|if cmd\|command ==\|def " python/pyreverseetl/cli.py | head -60; ls <host-path>
sudo DEBIAN_FRONTEND=noninteractive apt-get install -y -qq postgresql >/tmp/pg.log 2>&1; echo apt=$?; sudo systemctl start postgresql 2>&1; sudo -u postgres psql -tc "select version()" | head -2; sudo -u postgres psql -c "ALTER USER postgres PASSWORD 'postgres';" -c "CREATE DATABASE demo;" 2>&1
cd /tmp && sudo -u postgres psql -d demo -c "
  CREATE TABLE customers (id INT PRIMARY KEY, name TEXT, email TEXT, ltv NUMERIC);
  INSERT INTO customers VALUES
    (1, 'Alice', <email>', 4200.50),
    (2, 'Bob',   <email>',   1800.00);" 2>&1
cat > /tmp/hook_receiver.py <<'EOF'
import http.server
class H(http.server.BaseHTTPRequestHandler):
    def do_POST(self):
        n = int(self.headers.get('Content-Length', 0))
        body = self.rfile.read(n)
        with open('/tmp/hook_received.log', 'ab') as f: f.write(body + b'\n')
        self.send_response(200); self.end_headers(); self.wfile.write(b'ok')
    def log_message(self, *a): pass
http.server.HTTPServer(('127.0.0.1', 8000), H).serve_forever()
EOF
nohup python3 -I /tmp/hook_receiver.py > /tmp/hook.out 2>&1 &
sleep 1; curl -s -X POST -d '{"t":1}' http://localhost:8000/hook; echo; sed -n 30,45p <host-path>
cd /tmp && export PATH=<host-path>; export PYREVERSEETL_STATE_PATH=/tmp/pyreverseetl-probe/state.json; mkdir -p /tmp/pyreverseetl-probe; rm -f /tmp/hook_received.log
pyreverseetl create-workflow ltv_sync "LTV to webhook" postgres customers --source-config '{"host":"localhost","port":5432,"database":"demo","username":"postgres","password":"<secret>"}'; echo rc=$?
pyreverseetl create-activation ltv_to_hook ltv_sync webhook --dest-config '{"url":"http://localhost:8000/hook","auth":<secret>"type":"bearer","token":"<secret>"}}'; echo rc=$?
pyreverseetl execute ltv_to_hook; echo rc=$?
pyreverseetl lineage | cut -c1-600; echo rc=$?; echo ---; cat /tmp/hook_received.log 2>/dev/null | head -c 400
cd <host-path> && export PYREVERSEETL_STATE_PATH=/tmp/pyreverseetl-probe/state.json; <host-path> execute ltv_to_hook --dry-run | cut -c1-900; echo rc=${PIPESTATUS[0]}
pkill -f hook_receiver.py; sleep 1; pgrep -fa hook_receiver || echo receiver_stopped; mkdir -p <host-path> && ls -la <host-path>
cat > <host-path> <<'EOF'
{
  "kind": "cli",
  "argv": ["<host-path>", "execute", "ltv_to_hook", "--dry-run"],
  "cwd": "<host-path>",
  "env": {"PYREVERSEETL_STATE_PATH": "/tmp/pyreverseetl-probe/state.json"}
}
EOF
cat > <host-path> <<'EOF'
{
  "outcome": "working",
  "operation": "README Quick start: create-workflow (postgres customers) and create-activation (webhook), then `pyreverseetl execute ltv_to_hook --dry-run` (README Dry run section). Full execute to a live webhook was also run once and delivered 2 rows.",
  "operation_scope": "Postgres source is a local apt-installed PostgreSQL 16 instead of the docker container in the README. Dry run needs only Postgres; the real webhook POST was verified with a temporary local receiver that has been stopped. The probe's state file lives at /tmp/pyreverseetl-probe/state.json and needs Postgres running on localhost:5432.",
  "steps": [
    "apt-get install python3-venv python3-pip build-essential pkg-config libssl-dev curl ca-certificates (guest prerequisite)",
    "rustup install 1.97 (rust-toolchain.toml pin; minimal profile)",
    "python3 -m venv <host-path>; pip install maturin pytest",
    "apt-get install libsasl2-dev cmake zlib1g-dev libcurl4-openssl-dev (needed by sasl2-sys and rdkafka-sys build scripts)",
    "pip install patchelf (into venv; maturin needs it on PATH)",
    "CARGO_PROFILE_RELEASE_LTO=false CARGO_PROFILE_RELEASE_CODEGEN_UNITS=16 CARGO_BUILD_JOBS=2 maturin build --release -o /tmp/wheels (succeeded after the steps above)",
    "pip install /tmp/wheels/pyreverseetl-3.1.5-*.whl; pyreverseetl --version -> 3.1.5",
    "apt-get install postgresql; seeded demo.customers (2 rows)",
    "pyreverseetl create-workflow ltv_sync ...; create-activation ltv_to_hook ...; execute ltv_to_hook -> rows_synced 2, webhook received 2 POSTs",
    "pyreverseetl execute ltv_to_hook --dry-run -> rows_read 2, rows_would_write 2 (probe)"
  ],
  "deviations": [
    "README 'Installation' uses pip install pyreverseetl from PyPI; instead built the pinned checkout with maturin (the task requires the pinned source).",
    "Rust build with default release profile (LTO, codegen-units=1) was not attempted to completion; first `maturin build --release` failed with missing libsasl2 (sasl2-sys), then missing cmake (rdkafka-sys). Built with CARGO_PROFILE_RELEASE_LTO=false and CARGO_PROFILE_RELEASE_CODEGEN_UNITS=16 to keep the 4 GB guest from running out of memory; the aws-sdk-s3 compile with default settings failed in this guest, cause not captured in full.",
    "README Quick start docker run postgres:16 was not available (no docker). Installed PostgreSQL 16 via apt, created the postgres password and demo DB, and seeded customers with the README's SQL.",
    "README Quick start webhook target (python3 -m http.server) does not accept POST; used a temporary local POST receiver on 127.0.0.1:8000 for the full execute, then stopped it.",
    "Set PYREVERSEETL_STATE_PATH=/tmp/pyreverseetl-probe/state.json so workflow state is kept out of the workspace.",
    "Observed: the ltv NUMERIC column arrives as null in the webhook payload and dry-run preview. Lineage output was empty in a separate process (lineage is in-process per README-described behaviour), so lineage was not used as evidence."
  ],
  "stuck_step": null,
  "main_blocker": null,
  "suggested_fix": null
}
EOF
cd <host-path> && <env-var>=<host-path> python3 -I <host-path> --validate; echo rc=$?
pgrep -fa hook_receiver.py | grep -v pgrep || echo receiver_stopped; ls -la <host-path> 2>&1; systemctl is-active postgresql
mkdir -p <host-path> && cat > <host-path> <<'EOF'
{
  "kind": "cli",
  "argv": ["<host-path>", "execute", "ltv_to_hook", "--dry-run"],
  "cwd": "<host-path>",
  "env": {"PYREVERSEETL_STATE_PATH": "/tmp/pyreverseetl-probe/state.json"}
}
EOF
cat > <host-path> <<'EOF'
{
  "outcome": "working",
  "operation": "README Quick start: create-workflow (postgres customers) and create-activation (webhook), then `pyreverseetl execute ltv_to_hook --dry-run` (README Dry run section). A full execute to a live webhook was also run once and delivered 2 rows.",
  "operation_scope": "Postgres source is a local apt-installed PostgreSQL 16 instead of the README's docker container. The dry run needs only Postgres on localhost:5432 with the demo DB seeded. The real webhook POST was verified with a temporary local receiver, which has been stopped.",
  "steps": [
    "apt-get install python3-venv python3-pip build-essential pkg-config libssl-dev curl ca-certificates",
    "rustup install 1.97 (rust-toolchain.toml pin, minimal profile)",
    "python3 -m venv <host-path>; pip install maturin pytest",
    "apt-get install libsasl2-dev cmake zlib1g-dev libcurl4-openssl-dev (needed by sasl2-sys and rdkafka-sys build scripts)",
    "pip install patchelf into the venv, and put the venv bin on PATH for maturin",
    "CARGO_PROFILE_RELEASE_LTO=false CARGO_PROFILE_RELEASE_CODEGEN_UNITS=16 CARGO_BUILD_JOBS=2 maturin build --release -o /tmp/wheels",
    "pip install /tmp/wheels/pyreverseetl-3.1.5-*.whl; pyreverseetl --version prints 3.1.5",
    "apt-get install postgresql; seeded demo.customers with 2 rows (README SQL)",
    "pyreverseetl create-workflow ltv_sync ...; create-activation ltv_to_hook ...; execute ltv_to_hook -> rows_synced 2, receiver got 2 POSTs",
    "pyreverseetl execute ltv_to_hook --dry-run -> rows_read 2, rows_would_write 2 (probe)"
  ],
  "deviations": [
    "README Installation uses pip install pyreverseetl from PyPI. The pinned checkout was built with maturin instead, as the task requires.",
    "Default release build was not used. The first maturin build failed on missing libsasl2 (sasl2-sys), then missing cmake (rdkafka-sys). The build then used CARGO_PROFILE_RELEASE_LTO=false and CARGO_PROFILE_RELEASE_CODEGEN_UNITS=16 to avoid the 4 GB guest running out of memory. An earlier default-profile aws-sdk-s3 compile failed, and I did not capture its full error.",
    "README Quick start docker run postgres:16 was unavailable (no docker). Installed PostgreSQL 16 via apt, set the postgres password, created the demo DB, and seeded customers with the README SQL.",
    "README Quick start webhook target (python3 -m http.server) does not accept POST. Used a temporary POST receiver on 127.0.0.1:8000 for the full execute, then stopped it.",
    "Set PYREVERSEETL_STATE_PATH=/tmp/pyreverseetl-probe/state.json so workflow state stays out of the workspace.",
    "Observed: the NUMERIC ltv column arrives as null in both the webhook payload and the dry-run preview. Lineage was empty when queried from a separate process, so it was not used as evidence."
  ],
  "stuck_step": null,
  "main_blocker": null,
  "suggested_fix": null
}
EOF
cd <host-path> && <env-var>=<host-path> python3 -I <host-path> --validate; echo rc=$?