OTA tutorial — companion files

BBPATH .= ":${LAYERDIR}"
BBFILES += "${LAYERDIR}/recipes-*/*/*.bb ${LAYERDIR}/recipes-*/*/*.bbappend"
BBFILE_COLLECTIONS += "ota-demo"
BBFILE_PATTERN_ota-demo = "^${LAYERDIR}/"
BBFILE_PRIORITY_ota-demo = "20"
LAYERDEPENDS_ota-demo = "core"
LAYERSERIES_COMPAT_ota-demo = "scarthgap wrynose whinlatter"

SUMMARY = "OTA demo version marker (/etc/ota-demo-version)"
LICENSE = "MIT"
LIC_FILES_CHKSUM = "file://${COMMON_LICENSE_DIR}/MIT;md5=0835ade698e0bcf8506ecda2f7b4f302"

# Bump OTA_DEMO_VERSION to publish a new visible version.
OTA_DEMO_VERSION ?= "1"

inherit allarch

do_install() {
    install -d ${D}${sysconfdir}
    echo "${OTA_DEMO_VERSION}" > ${D}${sysconfdir}/ota-demo-version
}

FILES:${PN} = "${sysconfdir}/ota-demo-version"
CONFFILES:${PN} = "${sysconfdir}/ota-demo-version"

# OTA demo: override the desktop wallpaper with our image.
# FILESEXTRAPATHS:prepend, plus the layer's higher BBFILE_PRIORITY, so our
# qcom-background.png wins.
FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"

SUMMARY = "QLI 2.0 OSTree OTA update agent (web UI)"
DESCRIPTION = "A tiny dependency-free Python web service that drives ostree \
pull/deploy/reboot from a button click. POC companion for the QLI 2.0 OTA tutorial."
LICENSE = "BSD-3-Clause"
LIC_FILES_CHKSUM = "file://${COMMON_LICENSE_DIR}/BSD-3-Clause;md5=550794465ba0ec5312d6919e203a55f9"

SRC_URI = " \
    file://ota-agent.py \
    file://ota-agent.service \
    file://ota-agent.conf \
"

# Files unpack here (modern Yocto: S = ${WORKDIR} is no longer allowed).
S = "${UNPACKDIR}"

# The agent is pure Python stdlib. On Yocto the stdlib is split into many
# packages, so we must pull in exactly the ones the agent imports:
#   python3-html      -> html, html.parser   (html.escape in the UI)
#   python3-netclient -> http.server
#   python3-netserver -> socketserver
#   python3-json      -> json
#   python3-core      -> os, re, subprocess, threading, urllib.request
# (verified against build/tmp/pkgdata, not guessed). ostree is the update tool.
RDEPENDS:${PN} = "python3-core python3-html python3-netclient python3-netserver python3-json ostree"

inherit systemd

SYSTEMD_SERVICE:${PN} = "ota-agent.service"
SYSTEMD_AUTO_ENABLE = "enable"

do_install() {
    install -d ${D}${bindir}
    install -m 0755 ${S}/ota-agent.py ${D}${bindir}/ota-agent.py

    install -d ${D}${sysconfdir}
    install -m 0644 ${S}/ota-agent.conf ${D}${sysconfdir}/ota-agent.conf

    install -d ${D}${systemd_system_unitdir}
    install -m 0644 ${S}/ota-agent.service ${D}${systemd_system_unitdir}/ota-agent.service
}

FILES:${PN} += " \
    ${bindir}/ota-agent.py \
    ${sysconfdir}/ota-agent.conf \
    ${systemd_system_unitdir}/ota-agent.service \
"

# Config is editable on-device and must survive OSTree updates; ship it as a conffile.
CONFFILES:${PN} += "${sysconfdir}/ota-agent.conf"

[Unit]
Description=QLI 2.0 OSTree OTA update agent (web UI)
After=network-online.target
Wants=network-online.target

[Service]
Type=simple
ExecStart=/usr/bin/python3 /usr/bin/ota-agent.py
Restart=on-failure
RestartSec=3

[Install]
WantedBy=multi-user.target

# ota-agent configuration. Edit SERVER_URL to point at your update server.
# The device fetches <SERVER_URL>/version.json and pulls <SERVER_URL>/<OSTREE_REPO_PATH>.

# Update server on the LAN:
SERVER_URL=http://CHANGE_ME:8000

# Path of the served ostree repo, relative to SERVER_URL:
OSTREE_REPO_PATH=ostree_repo

# Name the agent registers the ostree remote under:
REMOTE_NAME=otaserver

# Port the agent's own web UI listens on:
LISTEN_PORT=8088

#!/usr/bin/env python3
"""
ota-agent — a tiny, dependency-free OTA update agent for Qualcomm Linux (QLI 2.0).

Runs an HTTP server ON the device. It:
  * shows the current booted OSTree deployment and the version available on the
    update server (read from <server>/version.json),
  * on a button click: `ostree pull` the new commit over HTTP, `ostree admin
    deploy` it, then reboot into it.

OSTree gives us atomic deploy + automatic rollback (systemd boot-counting) for
free, so this agent stays deliberately small. Only the Python 3 standard library
is used so it runs on any QLI image without extra packages.

Config is read from /etc/ota-agent.conf (simple KEY=VALUE), with env-var and
sane-default fallbacks so it also runs straight from a checkout for testing.

Companion to the QLI 2.0 OTA tutorial. Proof of concept: plain HTTP, no signing.
"""

import http.server
import socketserver
import json
import os
import re
import subprocess
import threading
import urllib.request
import html

# --------------------------------------------------------------------------- #
# Configuration
# --------------------------------------------------------------------------- #
CONF_PATH = os.environ.get("OTA_AGENT_CONF", "/etc/ota-agent.conf")


def load_config():
    cfg = {
        # Where the update server lives. version.json + the ostree repo are served here.
        "SERVER_URL": "http://127.0.0.1:8000",
        # Sub-path (under SERVER_URL) of the served ostree repo.
        "OSTREE_REPO_PATH": "ostree_repo",
        # Name to register the remote under on the device.
        "REMOTE_NAME": "otaserver",
        # Port the agent's own web UI listens on.
        "LISTEN_PORT": "8088",
    }
    try:
        with open(CONF_PATH) as fh:
            for line in fh:
                line = line.strip()
                if not line or line.startswith("#") or "=" not in line:
                    continue
                k, v = line.split("=", 1)
                cfg[k.strip()] = v.strip().strip('"').strip("'")
    except FileNotFoundError:
        pass
    # env overrides
    for k in list(cfg):
        if k in os.environ:
            cfg[k] = os.environ[k]
    return cfg


CFG = load_config()

# A single in-memory job so we never run two updates at once and the UI can poll status.
JOB = {"state": "idle", "log": [], "lock": threading.Lock()}


def log(msg):
    JOB["log"].append(msg)
    print("[ota-agent]", msg, flush=True)


# --------------------------------------------------------------------------- #
# OSTree helpers
# --------------------------------------------------------------------------- #
def run(cmd, timeout=600):
    """Run a command, return (rc, combined_output)."""
    log("$ " + " ".join(cmd))
    try:
        p = subprocess.run(
            cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
            timeout=timeout, text=True,
        )
        if p.stdout:
            for ln in p.stdout.rstrip().splitlines():
                log("  " + ln)
        return p.returncode, p.stdout or ""
    except FileNotFoundError:
        log("  ! command not found: %s" % cmd[0])
        return 127, "command not found"
    except subprocess.TimeoutExpired:
        log("  ! timed out")
        return 124, "timeout"


def ostree_status():
    """Parse `ostree admin status` into the current commit + ref + rollback info."""
    rc, out = run(["ostree", "admin", "status"], timeout=30)
    if rc != 0:
        return {"available": False, "raw": out}
    current = {"commit": None, "ref": None, "version": None}
    rollback = None
    cur_block = None
    for line in out.splitlines():
        m = re.match(r"^\s*(\*?)\s*(\S+)\s+([0-9a-f]{8,})(\.\d+)?(\s+\(rollback\))?\s*$", line)
        if m:
            star, osname, commit, _idx, rb = m.groups()
            block = {"osname": osname, "commit": commit}
            if star == "*":
                current.update(block)
                cur_block = current
            elif rb:
                rollback = block
            cur_block = current if star == "*" else block
            continue
        mv = re.match(r"^\s*Version:\s*(.+?)\s*$", line)
        if mv and cur_block is not None:
            cur_block["version"] = mv.group(1)
        mo = re.match(r"^\s*origin refspec:\s*(.+?)\s*$", line)
        if mo and cur_block is not None:
            cur_block["ref"] = mo.group(1)
    return {"available": True, "raw": out, "current": current, "rollback": rollback}


def ostree_refs():
    rc, out = run(["ostree", "refs"], timeout=30)
    if rc != 0:
        return []
    return [r.strip() for r in out.splitlines() if r.strip()]


def remote_version():
    """Fetch version.json from the update server."""
    url = CFG["SERVER_URL"].rstrip("/") + "/version.json"
    try:
        with urllib.request.urlopen(url, timeout=10) as r:
            return json.loads(r.read().decode())
    except Exception as e:  # noqa: BLE001 - surface any fetch error in the UI
        return {"error": str(e), "url": url}


# --------------------------------------------------------------------------- #
# The update job
# --------------------------------------------------------------------------- #
def do_update():
    if not JOB["lock"].acquire(blocking=False):
        return
    try:
        JOB["state"] = "running"
        JOB["log"] = []
        meta = remote_version()
        if "error" in meta:
            log("Cannot reach update server: %s" % meta["error"])
            JOB["state"] = "error"
            return
        ref = meta.get("ref")
        repo_url = CFG["SERVER_URL"].rstrip("/") + "/" + CFG["OSTREE_REPO_PATH"].strip("/")
        remote = CFG["REMOTE_NAME"]

        log("Target version %s (ref %s)" % (meta.get("version"), ref))

        # (Re)register the remote, no GPG for the POC.
        run(["ostree", "remote", "delete", "--if-exists", remote], timeout=30)
        rc, _ = run(["ostree", "remote", "add", "--no-gpg-verify", remote, repo_url], timeout=30)
        if rc != 0:
            JOB["state"] = "error"; return

        rc, _ = run(["ostree", "pull", remote, ref], timeout=1800)
        if rc != 0:
            log("pull failed"); JOB["state"] = "error"; return

        rc, _ = run(["ostree", "admin", "deploy", "%s:%s" % (remote, ref)], timeout=600)
        if rc != 0:
            log("deploy failed"); JOB["state"] = "error"; return

        log("Deploy staged. Rebooting into the new version in 3s ...")
        JOB["state"] = "rebooting"
        threading.Timer(3.0, lambda: subprocess.run(["reboot"])).start()
    finally:
        JOB["lock"].release()


# --------------------------------------------------------------------------- #
# Web UI
# --------------------------------------------------------------------------- #
PAGE = """<!doctype html>
<html><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1">
<title>QLI 2.0 OTA Update</title>
<style>
 body{{font-family:-apple-system,Segoe UI,Roboto,sans-serif;max-width:760px;margin:2rem auto;padding:0 1rem;color:#1a1a2e}}
 h1{{font-size:1.4rem}} .card{{border:1px solid #ddd;border-radius:10px;padding:1rem 1.25rem;margin:1rem 0}}
 .v{{font-size:1.6rem;font-weight:700}} .muted{{color:#666;font-size:.85rem}}
 .ok{{color:#0a7d2c}} .new{{color:#b34700}} code{{background:#f3f3f7;padding:.1rem .3rem;border-radius:4px}}
 button{{background:#5a31f4;color:#fff;border:0;border-radius:8px;padding:.7rem 1.3rem;font-size:1rem;cursor:pointer}}
 button:disabled{{background:#aaa;cursor:not-allowed}}
 pre{{background:#0e0e1a;color:#cfe;padding:1rem;border-radius:8px;overflow:auto;max-height:320px;font-size:.8rem}}
 .pill{{display:inline-block;padding:.15rem .6rem;border-radius:999px;font-size:.8rem;background:#eee}}
</style></head><body>
<h1>Qualcomm Linux 2.0 &mdash; OSTree OTA Update</h1>
<div class="card">
  <div class="muted">RUNNING ON THIS DEVICE</div>
  <div class="v">{cur_version}</div>
  <div class="muted">commit <code>{cur_commit}</code><br>ref <code>{cur_ref}</code></div>
  {rollback_html}
</div>
<div class="card">
  <div class="muted">AVAILABLE ON UPDATE SERVER <span class="pill">{server}</span></div>
  <div class="v {avail_class}">{avail_version}</div>
  <div class="muted">{avail_detail}</div>
  <p>{action_html}</p>
</div>
{log_html}
<p class="muted">ostree status:</p><pre>{status_raw}</pre>
</body></html>"""


class Handler(http.server.BaseHTTPRequestHandler):
    def _send(self, code, body, ctype="text/html; charset=utf-8"):
        b = body.encode() if isinstance(body, str) else body
        self.send_response(code)
        self.send_header("Content-Type", ctype)
        self.send_header("Content-Length", str(len(b)))
        self.end_headers()
        self.wfile.write(b)

    def log_message(self, *a):  # quiet
        pass

    def do_GET(self):
        if self.path.startswith("/status.json"):
            self._send(200, json.dumps({
                "job": JOB["state"], "log": JOB["log"],
                "ostree": ostree_status(), "remote": remote_version(),
            }), "application/json")
            return
        if self.path != "/" and not self.path.startswith("/?"):
            self._send(404, "not found")
            return
        self._send(200, self.render())

    def do_POST(self):
        if self.path.startswith("/update"):
            threading.Thread(target=do_update, daemon=True).start()
            self.send_response(303); self.send_header("Location", "/"); self.end_headers()
            return
        self._send(404, "not found")

    def render(self):
        st = ostree_status()
        rv = remote_version()
        cur = st.get("current", {}) if st.get("available") else {}
        cur_commit_full = cur.get("commit") or ""
        cur_commit = cur_commit_full[:16] or "?"
        cur_ref = cur.get("ref") or "?"
        cur_version = cur.get("version") or "(unknown)"

        rollback_html = ""
        if st.get("rollback"):
            rollback_html = '<div class="muted">rollback available: <code>%s</code></div>' % st["rollback"]["commit"][:16]

        if "error" in rv:
            avail_version = "unreachable"
            avail_detail = "Could not fetch %s &mdash; %s" % (html.escape(rv.get("url", "")), html.escape(rv["error"]))
            avail_class = ""
            action_html = '<button disabled>Update unavailable</button>'
        else:
            avail_version = html.escape(str(rv.get("version", "?")))
            avail_commit_full = str(rv.get("commit", ""))
            avail_detail = "ref <code>%s</code> &middot; commit <code>%s</code><br>%s" % (
                html.escape(str(rv.get("ref", "?"))),
                html.escape(avail_commit_full[:16]),
                html.escape(str(rv.get("notes", ""))),
            )
            # Compare by commit hash — the reliable identity. The version field is
            # just a human label (DISTRO_VERSION is "2.0" for every build, so it
            # can't distinguish releases).
            same = bool(cur_commit_full) and bool(avail_commit_full) and (cur_commit_full == avail_commit_full)
            avail_class = "ok" if same else "new"
            if JOB["state"] in ("running", "rebooting"):
                action_html = '<button disabled>Updating&hellip;</button>'
            elif same:
                action_html = '<span class="ok">&#10003; Device is up to date.</span>'
            else:
                action_html = ('<form method="post" action="/update">'
                               '<button type="submit">Download &amp; install %s, then reboot</button></form>'
                               % avail_version)

        log_html = ""
        if JOB["log"]:
            log_html = '<div class="card"><div class="muted">UPDATE LOG (%s)</div><pre>%s</pre></div>' % (
                JOB["state"], html.escape("\n".join(JOB["log"][-200:])))

        return PAGE.format(
            cur_version=html.escape(str(cur_version)),
            cur_commit=html.escape(str(cur_commit)),
            cur_ref=html.escape(str(cur_ref)),
            rollback_html=rollback_html,
            server=html.escape(CFG["SERVER_URL"]),
            avail_version=avail_version, avail_detail=avail_detail,
            avail_class=avail_class, action_html=action_html,
            log_html=log_html,
            status_raw=html.escape(st.get("raw", "")),
        )


class ThreadingServer(socketserver.ThreadingMixIn, http.server.HTTPServer):
    daemon_threads = True
    allow_reuse_address = True


def main():
    port = int(CFG["LISTEN_PORT"])
    log("ota-agent starting on :%d, server=%s" % (port, CFG["SERVER_URL"]))
    ThreadingServer(("0.0.0.0", port), Handler).serve_forever()


if __name__ == "__main__":
    main()

header:
  version: 14
repos:
  meta-ota-demo:
    path: meta-ota-demo
local_conf_header:
  ota-agent: |
    IMAGE_INSTALL:append = " ota-agent ota-demo-version vim"

#!/usr/bin/env bash
# publish.sh — publish a freshly built QLI image as an OTA version.
#
# Copies the ostree_repo produced by the SOTA build into the update server's
# web root (so the device's ota-agent can pull it), then writes version.json
# describing the new release.
#
# Usage:
#   ./publish.sh <VERSION> ["release notes"]
# Example:
#   ./publish.sh 2 "v2: marker=2, wallpaper B, vim"
#
# Companion to the QLI 2.0 OTA tutorial.
set -euo pipefail

WWW="$HOME/ota-server/www"
REPO_DST="$WWW/ostree_repo"
MACHINE="${MACHINE:-iq-8275-evk}"
# Where the SOTA build dropped its output. Edit to match your build host.
DEPLOY_DIR="${DEPLOY_DIR:-$HOME/yocto-build/build/tmp/deploy/images/${MACHINE}}"
SRC_REPO="${DEPLOY_DIR}/ostree_repo"

VERSION="${1:?usage: publish.sh <VERSION> [notes]}"
NOTES="${2:-}"

mkdir -p "$WWW"
echo "==> Syncing ostree_repo from ${SRC_REPO}"
# --delete keeps the served repo identical to the build output.
# If the build host is a different machine, change this to pull over SSH:
#   rsync -az --delete user@BUILD_HOST:"${SRC_REPO}/" "${REPO_DST}/"
rsync -a --delete "${SRC_REPO}/" "${REPO_DST}/"

REF="$(find "${REPO_DST}/refs/heads" -type f | sed "s#${REPO_DST}/refs/heads/##" | head -1)"
COMMIT="$(cat "${REPO_DST}/refs/heads/${REF}")"

cat > "$WWW/version.json" <<JSON
{
  "version": "${VERSION}",
  "ref": "${REF}",
  "commit": "${COMMIT}",
  "machine": "${MACHINE}",
  "notes": "${NOTES}"
}
JSON

echo "==> Published version ${VERSION}"
echo "    ref:    ${REF}"
echo "    commit: ${COMMIT}"
echo "    notes:  ${NOTES}"
echo "    www/version.json written. Make sure the HTTP server is running."