New Database

This commit is contained in:
喆OvO
2026-08-11 22:07:55 +08:00
parent cbbb2e6d34
commit ceb4404c6a
28 changed files with 1825 additions and 768 deletions
-1
View File
@@ -84,7 +84,6 @@ def delete_alarm(alarm_id: int, user_id: int = None) -> bool:
def check_alarms(user_id: int = None) -> list:
if user_id is None:
user_id = _uid()
global _triggered_state
db = get_db(current_app.config["DB_PATH"])
now = tz_now()
+62 -30
View File
@@ -1,16 +1,12 @@
"""Flask application factory and SSE event system."""
import json
import os
import queue
import secrets
import sqlite3
import threading
from datetime import datetime, timedelta
from datetime import timedelta
from backend.time_util import now as tz_now, strptime as tz_strptime
from flask import Flask
from flask_cors import CORS
from werkzeug.middleware.proxy_fix import ProxyFix
from backend.models import init_db
from backend.models import init_db, get_db
from backend.routes import register_routes
from backend.events import EventBus
from backend.weather_mgr import fetch_weather, fetch_hourly
@@ -70,33 +66,69 @@ def create_app(db_path: str = "data.db") -> Flask:
return app
def _startup_weather(app, db_path: str):
try:
conn = sqlite3.connect(db_path)
row = conn.execute(
"SELECT updated_at FROM weather_cache WHERE user_id=0 ORDER BY id DESC LIMIT 1"
).fetchone()
conn.close()
except Exception:
row = None
# Startup weather fetch lock: only one Gunicorn worker may fetch at a time
# (others skip; the cache is then fresh so the regular scheduler job handles it).
_WEATHER_STARTUP_LOCK_KEY = 0x5745415448455253 # "WEATHERS"
should_fetch = True
if row and row[0]:
def _startup_weather(app, db_path: str):
# Multi-worker guard: without this, every worker would hit the weather API
# concurrently on boot. Whoever wins the advisory lock runs the check/fetch.
lock_conn = None
got_lock = False
try:
import psycopg2
lock_conn = psycopg2.connect(db_path, connect_timeout=5)
cur = lock_conn.cursor()
cur.execute("SELECT pg_try_advisory_lock(%s)", (_WEATHER_STARTUP_LOCK_KEY,))
got_lock = bool(cur.fetchone()[0])
cur.close()
except Exception:
if lock_conn is not None:
try:
lock_conn.close()
except Exception:
pass
return
if not got_lock:
try:
updated = tz_strptime(row[0], "%Y-%m-%d %H:%M:%S")
age = tz_now() - updated
if age < timedelta(minutes=30):
should_fetch = False
app.logger.info(
f"Startup: weather cache is {int(age.total_seconds()/60)} min old, skip fetch"
)
lock_conn.close()
except Exception:
pass
return
if should_fetch:
try:
try:
fetch_weather()
fetch_hourly()
app.logger.info("Startup: weather fetched (cache stale or empty)")
except Exception as e:
app.logger.warning(f"Startup weather fetch failed: {e}")
conn = get_db(db_path)
row = conn.execute(
"SELECT updated_at FROM weather_cache WHERE user_id=0 ORDER BY id DESC LIMIT 1"
).fetchone()
conn.close()
except Exception:
row = None
should_fetch = True
if row and row["updated_at"]:
try:
updated = tz_strptime(row["updated_at"], "%Y-%m-%d %H:%M:%S")
age = tz_now() - updated
if age < timedelta(minutes=30):
should_fetch = False
app.logger.info(
f"Startup: weather cache is {int(age.total_seconds()/60)} min old, skip fetch"
)
except Exception:
pass
if should_fetch:
try:
fetch_weather()
fetch_hourly()
app.logger.info("Startup: weather fetched (cache stale or empty)")
except Exception as e:
app.logger.warning(f"Startup weather fetch failed: {e}")
finally:
try:
lock_conn.close() # releases the advisory lock
except Exception:
pass
+3 -4
View File
@@ -20,11 +20,10 @@ import string
import time
from flask import g, redirect, url_for, request, jsonify, current_app
from backend.models import (
get_auth_config, update_auth_config, get_temp_password, set_temp_password,
clear_temp_password, get_recovery_codes, set_recovery_codes, has_mfa_configured,
user_exists, is_admin_user, get_db,
get_auth_config, get_temp_password,
is_admin_user, get_db,
create_auth_token, find_access_token,
delete_auth_token, REFRESH_TOKEN_TTL, ACCESS_TOKEN_TTL,
delete_auth_token, REFRESH_TOKEN_TTL,
)
# ─── Token / cookie constants ────────────────
+1 -4
View File
@@ -4,10 +4,7 @@ Every function is scoped to an explicit `user_id` so each user only ever
sees/mutates their own data (defense in depth: rows are filtered by user_id
even though they also join user-scoped groups/sources).
"""
import os
import hashlib
import tempfile
from datetime import datetime, timedelta, date
from datetime import datetime, timedelta
from backend.time_util import now as tz_now
from flask import current_app
from backend.models import get_db
-1
View File
@@ -1,6 +1,5 @@
"""Countdown timer management — duration-based, once-only."""
import time
from datetime import datetime, timedelta
from backend.time_util import isoformat as tz_isoformat
from flask import current_app
from backend.models import get_db
+263
View File
@@ -0,0 +1,263 @@
"""Database administration: export / import / format detection.
Only Root may call these (enforced at the route layer). All operations are
atomic: an import runs inside a single transaction — any failing row rolls the
whole import back, leaving the current database untouched.
Formats:
- ``sqlite`` — a legacy SQLite database file (magic bytes ``SQLite format 3``).
Tables not present in the current SCHEMA are dropped (e.g. camera_config);
the legacy single-user `auth_config` row (keyed by id=1) is attributed to
Root (user 0); missing columns are filled with their SCHEMA DEFAULT.
- ``json`` — the self-contained backup produced by :func:`export_database`
(``{"format": "iot-dboard-backup", ...}``).
"""
import json
import os
import sqlite3
import tempfile
from datetime import datetime, timezone
from backend.models import (
get_db, SCHEMA, DB_VERSION, _CREATE_TABLE_RE, _table_columns,
_read_db_version, _ensure_defaults, _fix_identity_sequences,
)
# Tables whose contents are exported/imported (everything except the schema
# version row and transient session tokens).
_EXCLUDE_TABLES = {"meta", "auth_tokens"}
def _schema_table_names() -> list:
return [m.group(1) for m in _CREATE_TABLE_RE.finditer(SCHEMA)]
def detect_format(data: bytes) -> str:
"""Detect the database file format: 'sqlite', 'json' or 'unknown'."""
if data[:16] == b"SQLite format 3\x00":
return "sqlite"
try:
obj = json.loads(data.decode("utf-8"))
if isinstance(obj, dict) and obj.get("format") == "iot-dboard-backup":
return "json"
except Exception:
pass
return "unknown"
def export_database(db_path: str) -> dict:
"""Snapshot every data table (except auth_tokens) into a JSON-serializable dict."""
conn = get_db(db_path)
try:
tables = _schema_table_names()
out = {
"format": "iot-dboard-backup",
"backup_version": 1,
"db_version": _read_db_version(conn) or DB_VERSION,
"exported_at": datetime.now(timezone.utc).isoformat(),
"tables": {},
}
for t in tables:
if t in _EXCLUDE_TABLES:
continue
cols = _table_columns(conn, t)
if not cols:
continue
rows = conn.execute(
f"SELECT {', '.join(cols)} FROM {t} ORDER BY 1"
).fetchall()
out["tables"][t] = {
"columns": cols,
"rows": [[r[c] for c in cols] for r in rows],
}
return out
finally:
conn.close()
def export_database_json(db_path: str) -> str:
"""Export as a compact JSON string (files are small; no pretty printing)."""
return json.dumps(export_database(db_path), ensure_ascii=False, default=str)
# ─── SQLite import ───────────────────────────
def _sqlite_tables(conn_old) -> list:
return [
r["name"] for r in conn_old.execute(
"SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'"
).fetchall()
]
def _sqlite_rows(conn_old, table: str, cols: list) -> list:
if not cols:
return []
col_str = ", ".join(cols)
return conn_old.execute(f"SELECT {col_str} FROM {table}").fetchall()
def _read_sqlite(data: bytes) -> dict:
"""Read a legacy SQLite file into {table: {"columns": [...], "rows": [...]}}.
auth_config special case: legacy single-user row (id=1) belongs to Root."""
tmp_path = None
try:
fd, tmp_path = tempfile.mkstemp(suffix=".db")
with os.fdopen(fd, "wb") as f:
f.write(data)
conn_old = sqlite3.connect(f"file:{tmp_path}?mode=ro", uri=True)
conn_old.row_factory = sqlite3.Row
try:
tables = {}
for t in _sqlite_tables(conn_old):
cols = [r["name"] for r in conn_old.execute(f"PRAGMA table_info({t})").fetchall()]
if not cols:
continue
rows = []
for row in _sqlite_rows(conn_old, t, cols):
rows.append([row[c] for c in cols])
tables[t] = {"columns": cols, "rows": rows}
return tables
finally:
conn_old.close()
finally:
if tmp_path and os.path.exists(tmp_path):
os.unlink(tmp_path)
def _import_table(conn, table: str, data: dict, where: str, skipped: list) -> int:
"""Insert rows of one table into the target PG database (strict mode).
Returns the number of rows inserted."""
src_cols = data.get("columns") or []
rows = data.get("rows") or []
if not src_cols or not rows:
return 0
dst_cols = _table_columns(conn, table)
if not dst_cols:
skipped.append(table) # table not in schema — dropped (e.g. camera_config)
return 0
if table == "auth_config":
return _import_auth_config(conn, data, where)
# Only common columns are copied; missing target columns get their DEFAULT.
common = [c for c in dst_cols if c in src_cols]
if not common:
skipped.append(table)
return 0
col_str = ", ".join(common)
placeholders = ", ".join("?" * len(common))
sql = f"INSERT INTO {table} ({col_str}) VALUES ({placeholders})"
n = 0
for i, row in enumerate(rows):
vals = []
for c in common:
vals.append(row[src_cols.index(c)])
try:
conn.execute(sql, tuple(vals))
n += 1
except Exception as e:
raise ValueError(
f"{where}: 表 {table} 第 {i + 1} 行导入失败: {e}"
) from e
return n
def _import_auth_config(conn, data: dict, where: str) -> int:
"""Special case: the legacy auth_config table is keyed by `id` (single user,
id=1); the new table is keyed by `user_id`. The legacy row belongs to Root."""
src_cols = data.get("columns") or []
rows = data.get("rows") or []
if not src_cols or not rows:
return 0
dst_cols = _table_columns(conn, "auth_config")
n = 0
for i, row in enumerate(rows):
d = {c: row[src_cols.index(c)] for c in src_cols}
if "user_id" in src_cols:
uid = d.get("user_id") or 0
else:
old_id = d.get("id", 1)
uid = 0 if old_id == 1 else (old_id or 0)
common = [c for c in dst_cols if c in src_cols and c not in ("id", "user_id")]
if not common:
continue
col_str = ", ".join(common)
placeholders = ", ".join("?" * len(common))
sql = f"INSERT OR IGNORE INTO auth_config (user_id, {col_str}) VALUES (?, {placeholders})"
try:
conn.execute(sql, (uid, *[d[c] for c in common]))
n += 1
except Exception as e:
raise ValueError(
f"{where}: auth_config 第 {i + 1} 行导入失败: {e}"
) from e
return n
# ─── Restore core (shared by sqlite + json imports) ─────
def _restore(db_path: str, tables: dict, where: str) -> dict:
"""Atomic restore: TRUNCATE every data table, insert the given rows, then
re-bootstrap defaults and stamp the schema version. Any failure rolls back
the whole import (the shim rolls back on error, keeping it atomic)."""
conn = get_db(db_path)
stats: dict[str, int] = {}
skipped: list = []
try:
# 1. Wipe current data (TRUNCATE is transactional in PostgreSQL).
all_tables = _schema_table_names()
truncate_list = [t for t in all_tables if t != "meta"]
if truncate_list:
conn.execute("TRUNCATE TABLE " + ", ".join(truncate_list) + " CASCADE")
# 2. Insert data (strict: first failing row aborts everything).
for t in all_tables:
if t in _EXCLUDE_TABLES:
continue
if t in tables:
stats[t] = _import_table(conn, t, tables[t], where, skipped)
# 3. Re-bootstrap defaults (Root user, config rows) + stamp version.
_ensure_defaults(conn)
conn.execute("UPDATE meta SET version=? WHERE id=1", (DB_VERSION,))
conn.execute("INSERT OR IGNORE INTO meta (id, version) VALUES (1, ?)", (DB_VERSION,))
_fix_identity_sequences(conn)
conn.commit()
if skipped:
stats["_skipped_tables"] = skipped
return stats
except Exception:
try:
conn.rollback()
except Exception:
pass
raise
finally:
conn.close()
def import_sqlite(db_path: str, data: bytes) -> dict:
"""Import a legacy SQLite database file (atomic)."""
tables = _read_sqlite(data)
return _restore(db_path, tables, "SQLite 导入")
def import_json(db_path: str, data: dict) -> dict:
"""Import a JSON backup produced by export_database (atomic)."""
if not isinstance(data, dict) or data.get("format") != "iot-dboard-backup":
raise ValueError("不是有效的备份文件(缺少 iot-dboard-backup 标记)")
tables = data.get("tables") or {}
if not isinstance(tables, dict):
raise ValueError("备份文件结构无效:tables 缺失")
return _restore(db_path, tables, "备份导入")
def import_blob(db_path: str, data: bytes) -> dict:
"""Import a file, auto-detecting sqlite vs json backup."""
fmt = detect_format(data)
if fmt == "sqlite":
return import_sqlite(db_path, data)
if fmt == "json":
return import_json(db_path, json.loads(data.decode("utf-8")))
raise ValueError("无法识别的数据库文件格式(仅支持 SQLite 数据库或本系统导出的备份 JSON)")
-3
View File
@@ -44,6 +44,3 @@ class EventBus:
for item in dead:
self._subscribers.remove(item)
def broadcast(self):
"""Send heartbeat to all subscribers."""
self.publish("heartbeat", {"ts": __import__("time").time()})
+273 -200
View File
@@ -1,23 +1,31 @@
"""SQLite database models, initialization, and user-scoped helpers.
"""PostgreSQL database models, initialization, and user-scoped helpers.
Multi-user design:
Multi-user design (unchanged from the SQLite era):
- The `users` table stores every account. Root (ID 0) is the administrator and
is created automatically on first boot (fresh DB) or when migrating an
existing single-user database (all pre-existing data becomes Root's data).
- Every per-user table carries a `user_id INTEGER NOT NULL DEFAULT 0` column,
so legacy single-user rows are automatically attributed to Root on migration
so legacy single-user rows are automatically attributed to Root on import
and any NEW column added in the future gets a DEFAULT (the migration fills
missing columns with their SCHEMA defaults).
- `auth_config` is keyed by `user_id` (legacy `id=1` row maps to user 0).
The data-access layer talks PostgreSQL through ``backend.pg_compat``, a thin
sqlite3-compatible shim (``?`` placeholders, ``INSERT OR IGNORE``,
``datetime('now')``, ``cursor.lastrowid``, ``conn.total_changes`` are all
translated transparently).
"""
import sqlite3
import os
import re
import sys
import time
from backend.pg_compat import connect as _pg_connect
# ─── PostgreSQL schema ───────────────────────
# All new columns must carry a DEFAULT (migration contract: missing columns are
# filled with their SCHEMA default when the DB is upgraded).
SCHEMA = """
-- Global schema version (single row). All new columns must carry a DEFAULT.
-- Global schema version (single row).
CREATE TABLE IF NOT EXISTS meta (
id INTEGER PRIMARY KEY DEFAULT 1,
version INTEGER NOT NULL DEFAULT 1
@@ -26,7 +34,7 @@ CREATE TABLE IF NOT EXISTS meta (
-- Auth tokens (dual-token auth): access (short-lived, per-request) and
-- refresh (long-lived, stored in an httpOnly cookie). Only hashes are stored.
CREATE TABLE IF NOT EXISTS auth_tokens (
id INTEGER PRIMARY KEY AUTOINCREMENT,
id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
user_id INTEGER NOT NULL DEFAULT 0,
token_hash TEXT NOT NULL,
kind TEXT NOT NULL DEFAULT 'access' CHECK(kind IN ('access','refresh')),
@@ -34,14 +42,14 @@ CREATE TABLE IF NOT EXISTS auth_tokens (
temp INTEGER NOT NULL DEFAULT 0,
session_version INTEGER NOT NULL DEFAULT 0,
reauth_at INTEGER NOT NULL DEFAULT 0,
created_at TEXT DEFAULT (datetime('now')),
created_at TEXT DEFAULT (to_char(now(), 'YYYY-MM-DD HH24:MI:SS')),
expires_at INTEGER NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_auth_tokens_hash ON auth_tokens(token_hash);
CREATE INDEX IF NOT EXISTS idx_auth_tokens_user ON auth_tokens(user_id);
CREATE TABLE IF NOT EXISTS users (
id INTEGER PRIMARY KEY AUTOINCREMENT,
id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
remark TEXT NOT NULL DEFAULT '',
role TEXT NOT NULL DEFAULT 'user' CHECK(role IN ('admin','user')),
is_temp INTEGER NOT NULL DEFAULT 0,
@@ -49,7 +57,7 @@ CREATE TABLE IF NOT EXISTS users (
status TEXT NOT NULL DEFAULT 'active' CHECK(status IN ('active','pending')),
session_version INTEGER NOT NULL DEFAULT 0,
theme TEXT NOT NULL DEFAULT 'dark' CHECK(theme IN ('dark','light','auto')),
created_at TEXT DEFAULT (datetime('now'))
created_at TEXT DEFAULT (to_char(now(), 'YYYY-MM-DD HH24:MI:SS'))
);
CREATE TABLE IF NOT EXISTS auth_config (
@@ -64,16 +72,16 @@ CREATE TABLE IF NOT EXISTS auth_config (
);
CREATE TABLE IF NOT EXISTS calendar_groups (
id INTEGER PRIMARY KEY AUTOINCREMENT,
id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
user_id INTEGER NOT NULL DEFAULT 0,
name TEXT NOT NULL,
color TEXT DEFAULT '#4A90D9',
enabled INTEGER DEFAULT 1,
created_at TEXT DEFAULT (datetime('now'))
created_at TEXT DEFAULT (to_char(now(), 'YYYY-MM-DD HH24:MI:SS'))
);
CREATE TABLE IF NOT EXISTS calendar_sources (
id INTEGER PRIMARY KEY AUTOINCREMENT,
id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
user_id INTEGER NOT NULL DEFAULT 0,
group_id INTEGER NOT NULL,
type TEXT NOT NULL CHECK(type IN ('upload', 'subscription', 'manual')),
@@ -85,7 +93,7 @@ CREATE TABLE IF NOT EXISTS calendar_sources (
);
CREATE TABLE IF NOT EXISTS calendar_events (
id INTEGER PRIMARY KEY AUTOINCREMENT,
id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
user_id INTEGER NOT NULL DEFAULT 0,
source_id INTEGER NOT NULL,
uid TEXT,
@@ -99,7 +107,7 @@ CREATE TABLE IF NOT EXISTS calendar_events (
);
CREATE TABLE IF NOT EXISTS reminders (
id INTEGER PRIMARY KEY AUTOINCREMENT,
id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
user_id INTEGER NOT NULL DEFAULT 0,
calendar_group_id INTEGER NOT NULL,
minutes_before INTEGER DEFAULT 15,
@@ -108,7 +116,7 @@ CREATE TABLE IF NOT EXISTS reminders (
);
CREATE TABLE IF NOT EXISTS weather_config (
id INTEGER PRIMARY KEY AUTOINCREMENT,
id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
user_id INTEGER NOT NULL DEFAULT 0,
api_key TEXT DEFAULT '',
city TEXT DEFAULT 'Beijing',
@@ -124,35 +132,35 @@ CREATE TABLE IF NOT EXISTS weather_config (
);
CREATE TABLE IF NOT EXISTS weather_cache (
id INTEGER PRIMARY KEY AUTOINCREMENT,
id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
user_id INTEGER NOT NULL DEFAULT 0,
data TEXT,
updated_at TEXT DEFAULT (datetime('now'))
updated_at TEXT DEFAULT (to_char(now(), 'YYYY-MM-DD HH24:MI:SS'))
);
CREATE TABLE IF NOT EXISTS weather_hourly_cache (
id INTEGER PRIMARY KEY AUTOINCREMENT,
id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
user_id INTEGER NOT NULL DEFAULT 0,
data TEXT,
fetched_at TEXT DEFAULT (datetime('now'))
fetched_at TEXT DEFAULT (to_char(now(), 'YYYY-MM-DD HH24:MI:SS'))
);
CREATE TABLE IF NOT EXISTS weather_minutely_cache (
id INTEGER PRIMARY KEY AUTOINCREMENT,
id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
user_id INTEGER NOT NULL DEFAULT 0,
data TEXT,
fetched_at TEXT
);
CREATE TABLE IF NOT EXISTS weather_warning_cache (
id INTEGER PRIMARY KEY AUTOINCREMENT,
id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
user_id INTEGER NOT NULL DEFAULT 0,
data TEXT,
fetched_at TEXT
);
CREATE TABLE IF NOT EXISTS clock_config (
id INTEGER PRIMARY KEY AUTOINCREMENT,
id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
user_id INTEGER NOT NULL DEFAULT 0,
city TEXT DEFAULT 'Beijing',
timezone TEXT DEFAULT 'Asia/Shanghai',
@@ -160,7 +168,7 @@ CREATE TABLE IF NOT EXISTS clock_config (
);
CREATE TABLE IF NOT EXISTS alarms (
id INTEGER PRIMARY KEY AUTOINCREMENT,
id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
user_id INTEGER NOT NULL DEFAULT 0,
label TEXT DEFAULT '',
time TEXT NOT NULL,
@@ -171,7 +179,7 @@ CREATE TABLE IF NOT EXISTS alarms (
);
CREATE TABLE IF NOT EXISTS countdowns (
id INTEGER PRIMARY KEY AUTOINCREMENT,
id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
user_id INTEGER NOT NULL DEFAULT 0,
label TEXT DEFAULT '',
duration_seconds INTEGER DEFAULT 300,
@@ -181,7 +189,7 @@ CREATE TABLE IF NOT EXISTS countdowns (
);
CREATE TABLE IF NOT EXISTS display_config (
id INTEGER PRIMARY KEY AUTOINCREMENT,
id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
user_id INTEGER NOT NULL DEFAULT 0,
mode TEXT NOT NULL DEFAULT 'default' CHECK(mode IN ('default', 'custom')),
custom_html TEXT DEFAULT '',
@@ -191,7 +199,7 @@ CREATE TABLE IF NOT EXISTS display_config (
);
CREATE TABLE IF NOT EXISTS pomodoro_state (
id INTEGER PRIMARY KEY AUTOINCREMENT,
id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
user_id INTEGER NOT NULL DEFAULT 0,
running INTEGER DEFAULT 0,
phase TEXT DEFAULT 'work' CHECK(phase IN ('work', 'break')),
@@ -213,26 +221,27 @@ CREATE UNIQUE INDEX IF NOT EXISTS idx_pomodoro_state_user ON pomodoro_state(user
"""
# Tables whose rows are completely per-user (used for cascade delete).
USER_DATA_TABLES = [
"auth_config", "calendar_groups", "calendar_sources", "calendar_events",
"reminders", "weather_config", "weather_cache", "weather_hourly_cache",
"weather_minutely_cache", "weather_warning_cache", "clock_config",
"alarms", "countdowns", "display_config", "pomodoro_state",
# Ordered list of all user-data tables (children first) for FK-safe deletion.
_DELETE_ORDER = [
"calendar_events", "calendar_sources", "reminders", "calendar_groups",
"weather_cache", "weather_hourly_cache", "weather_minutely_cache",
"weather_warning_cache", "weather_config", "clock_config",
"alarms", "countdowns", "display_config", "pomodoro_state", "auth_config",
"auth_tokens",
]
def get_db(db_path: str = "data.db") -> sqlite3.Connection:
conn = sqlite3.connect(db_path)
conn.row_factory = sqlite3.Row
conn.execute("PRAGMA journal_mode=WAL")
conn.execute("PRAGMA foreign_keys=ON")
return conn
def get_db(db_path: str = "data.db"):
"""Open a PostgreSQL connection (sqlite3-compatible wrapper).
`db_path` is a PostgreSQL connection string (kept as the historical
parameter name to minimise churn across the codebase).
"""
return _pg_connect(db_path)
def _db_path(preferred: str = None) -> str:
"""Get the actual database path, preferring current_app config if available."""
"""Get the actual database connection string, preferring current_app config."""
if preferred:
return preferred
try:
@@ -330,16 +339,8 @@ def delete_user(user_id: int, db_path: str = None):
if user_id == 0:
return
conn = get_db(_db_path(db_path))
# Children first so FK constraints never reject a delete.
ordered = [
"calendar_events", "calendar_sources", "reminders", "calendar_groups",
"weather_cache", "weather_hourly_cache", "weather_minutely_cache",
"weather_warning_cache", "weather_config", "clock_config",
"alarms", "countdowns", "display_config", "pomodoro_state", "auth_config",
"auth_tokens",
]
try:
for table in ordered:
for table in _DELETE_ORDER:
conn.execute(f"DELETE FROM {table} WHERE user_id=?", (user_id,))
conn.execute("DELETE FROM users WHERE id=?", (user_id,))
conn.commit()
@@ -826,6 +827,14 @@ def find_user_by_credential_id(credential_id: str, db_path: str = None) -> int |
# ─── DB init / migration ─────────────────────
# Stores passwords that were just generated, for the startup script to print.
_EMITTED_PASSWORDS: list = []
# Current schema version supported by this program. Bump it whenever the
# SCHEMA changes so existing databases get upgraded on next start.
DB_VERSION = 2
def _ensure_defaults(conn):
"""Insert default rows for config tables and the Root user when missing.
Generates a random 6-digit temp password when Root's auth_config is created
@@ -843,6 +852,11 @@ def _ensure_defaults(conn):
conn.execute(
"INSERT OR IGNORE INTO users (id, remark, role, status) VALUES (0, 'Root', 'admin', 'active')"
)
# Keep the users identity sequence ahead of explicit id=0 insertion.
conn.execute(
"SELECT setval(pg_get_serial_sequence('users', 'id'), "
"GREATEST((SELECT COALESCE(MAX(id), 1) FROM users), 1))"
)
# weather_config for Root
count = conn.execute("SELECT COUNT(*) FROM weather_config WHERE user_id=0").fetchone()[0]
@@ -895,165 +909,224 @@ def _ensure_defaults(conn):
conn.commit()
# Stores passwords that were just generated, for the startup script to print.
_EMITTED_PASSWORDS: list = []
# Current schema version supported by this program. Bump it whenever the
# SCHEMA changes so existing databases get rebuilt+migrated on next start.
DB_VERSION = 1
def _table_columns(conn, table_name):
return [row["name"] for row in conn.execute(f"PRAGMA table_info({table_name})")]
def _migrate_auth_config(conn_new, conn_old):
"""Special-case migration for auth_config: the legacy table is keyed by
`id` (single user, id=1), the new table is keyed by `user_id`.
The legacy single-user row belongs to Root (user 0).
"""
old_cols = _table_columns(conn_old, "auth_config")
new_cols = _table_columns(conn_new, "auth_config")
if not old_cols:
return
rows = conn_old.execute("SELECT * FROM auth_config").fetchall()
for row in rows:
d = dict(row)
if "user_id" in old_cols:
uid = d.get("user_id") or 0
else:
# Legacy single-user DB: the id=1 row belongs to Root (user 0).
old_id = d.get("id", 1)
uid = 0 if old_id == 1 else (old_id or 0)
common = [c for c in new_cols if c in old_cols and c not in ("id", "user_id")]
if not common:
continue
col_str = ", ".join(common)
placeholders = ", ".join("?" * len(common))
conn_new.execute(
f"INSERT OR IGNORE INTO auth_config (user_id, {col_str}) VALUES (?, {placeholders})",
(uid, *[d[c] for c in common])
)
def init_db(db_path: str = "data.db"):
"""Initialize the database with schema-version aware migration.
- No database file: create it from the latest SCHEMA and create Root.
- Existing database:
* no `meta.version` (legacy DB) → rebuild + migrate compatible data
* DB version < DB_VERSION → rebuild + migrate
* DB version == DB_VERSION → use as-is (fast path)
* DB version > DB_VERSION → abort with an error (DB is newer
than this program)
Rebuild copies compatible data (legacy single-user data becomes Root's)
and fills missing columns with SCHEMA defaults.
"""
if not os.path.exists(db_path):
conn = get_db(db_path)
conn.executescript(SCHEMA)
_ensure_defaults(conn)
conn.execute("UPDATE meta SET version=? WHERE id=1", (DB_VERSION,))
conn.commit()
conn.close()
return
conn_probe = get_db(db_path)
old_version = _read_db_version(conn_probe)
conn_probe.close()
if old_version is not None and old_version > DB_VERSION:
raise SystemExit(
f"[FATAL] 数据库 schema 版本 (v{old_version}) 高于当前程序支持的版本 "
f"(v{DB_VERSION})。请先升级程序,或删除/降级数据库后再启动。"
)
if old_version is not None and old_version == DB_VERSION:
# Same version — no rebuild needed. Just ensure defaults exist.
conn = get_db(db_path)
_ensure_defaults(conn)
conn.execute("INSERT OR IGNORE INTO meta (id, version) VALUES (1, ?)", (DB_VERSION,))
conn.commit()
conn.close()
return
# Migrate by rebuilding: create a fresh DB with the latest schema,
# copy compatible data from the old DB, then replace the old DB.
new_db_path = db_path + ".new"
backup_path = db_path + ".backup"
# Clean up any stale temporary database from a previous failed run.
if os.path.exists(new_db_path):
os.remove(new_db_path)
conn_new = get_db(new_db_path)
conn_new.executescript(SCHEMA)
conn_old = get_db(db_path)
# Disable FK enforcement while copying; rows are copied table by table.
conn_old.execute("PRAGMA foreign_keys=OFF")
conn_new.execute("PRAGMA foreign_keys=OFF")
tables = conn_old.execute(
"SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'"
def _table_columns(conn, table_name: str) -> list:
"""Return the column names of a table (information_schema)."""
rows = conn.execute(
"SELECT column_name FROM information_schema.columns "
"WHERE table_schema = current_schema() AND table_name = ? "
"ORDER BY ordinal_position",
(table_name,)
).fetchall()
for (table_name,) in tables:
# Tables that no longer exist in the new schema are dropped (e.g. camera_config).
exists_new = conn_new.execute(
"SELECT 1 FROM sqlite_master WHERE type='table' AND name=?",
(table_name,)
).fetchone()
if not exists_new:
continue
if table_name == "auth_config":
_migrate_auth_config(conn_new, conn_old)
continue
old_cols = _table_columns(conn_old, table_name)
new_cols = _table_columns(conn_new, table_name)
common_cols = [c for c in new_cols if c in old_cols]
if not common_cols:
continue
col_str = ", ".join(common_cols)
placeholders = ", ".join("?" * len(common_cols))
sql = f"INSERT OR IGNORE INTO {table_name} ({col_str}) VALUES ({placeholders})"
for row in conn_old.execute(f"SELECT {col_str} FROM {table_name}"):
try:
conn_new.execute(sql, tuple(row))
except Exception:
# Ignore rows that cannot be migrated (e.g. constraint violations).
pass
conn_new.commit()
# Now fill defaults for any tables that are still empty + Root bootstrap.
_ensure_defaults(conn_new)
# Stamp the migrated database with the program's schema version.
conn_new.execute("UPDATE meta SET version=? WHERE id=1", (DB_VERSION,))
conn_new.commit()
conn_old.close()
conn_new.close()
# Replace old DB with the new one, keeping a backup.
if os.path.exists(backup_path):
os.remove(backup_path)
os.rename(db_path, backup_path)
os.rename(new_db_path, db_path)
return [r["column_name"] for r in rows]
def _read_db_version(conn) -> int | None:
"""Read the schema version from the `meta` table.
Returns None when the DB is legacy (no meta table / no version column).
Returns None when the DB is legacy (no meta table / no version row).
"""
try:
row = conn.execute("SELECT version FROM meta WHERE id=1").fetchone()
return int(row["version"]) if row else None
except Exception:
conn._safe_rollback()
return None
# ─── SCHEMA parsing for column-level migration ──────
_CREATE_TABLE_RE = re.compile(
r"CREATE TABLE IF NOT EXISTS\s+(\w+)\s*\((.*?)\)\s*;",
re.IGNORECASE | re.DOTALL,
)
def _split_top_level(text: str) -> list:
"""Split `text` on top-level commas (ignoring commas inside parentheses
AND inside single-quoted strings, e.g. DEFAULT '1,2,3,4,5,6,7')."""
parts = []
depth = 0
cur = []
in_str = False
i = 0
while i < len(text):
ch = text[i]
if ch == "'":
# SQL 单引号转义用两个单引号表示
if in_str and i + 1 < len(text) and text[i + 1] == "'":
cur.append("''")
i += 2
continue
in_str = not in_str
cur.append(ch)
elif not in_str:
if ch == "(":
depth += 1
elif ch == ")":
depth -= 1
if ch == "," and depth == 0:
parts.append("".join(cur))
cur = []
i += 1
continue
cur.append(ch)
else:
cur.append(ch)
i += 1
if cur:
parts.append("".join(cur))
return parts
def _strip_check(s: str) -> str:
"""Remove every CHECK(...) clause (parenthesis-balanced)."""
out = s
while True:
idx = out.upper().find("CHECK(")
if idx == -1:
break
depth = 0
end = -1
for j in range(idx, len(out)):
if out[j] == "(":
depth += 1
elif out[j] == ")":
depth -= 1
if depth == 0:
end = j
break
if end == -1:
break
out = out[:idx] + out[end + 1:]
return out
def _migrate_missing_columns(conn):
"""Add columns that exist in SCHEMA but are missing from existing tables.
New columns are created with their SCHEMA DEFAULT (migration contract).
Best-effort: a failing column does not block startup.
"""
for m in _CREATE_TABLE_RE.finditer(SCHEMA):
table = m.group(1)
try:
existing = set(_table_columns(conn, table))
except Exception:
conn._safe_rollback()
continue
for col_def in _split_top_level(m.group(2)):
col_def = col_def.strip()
if not col_def:
continue
upper = col_def.upper()
if upper.startswith(("PRIMARY KEY", "UNIQUE", "CHECK", "FOREIGN KEY", "CONSTRAINT")):
continue
name = col_def.split()[0].strip('"')
if not name or name in existing:
continue
# Make the ADD COLUMN safe for rows that already exist:
# drop NOT NULL / PK / UNIQUE / IDENTITY; keep TYPE + DEFAULT.
safe = re.sub(r"\s+NOT\s+NULL\b", "", col_def, flags=re.IGNORECASE)
safe = re.sub(r"\s+PRIMARY\s+KEY\b", "", safe, flags=re.IGNORECASE)
safe = re.sub(r"\s+UNIQUE\b", "", safe, flags=re.IGNORECASE)
safe = re.sub(r"\s+GENERATED\s+BY\s+DEFAULT\s+AS\s+IDENTITY\b", "", safe, flags=re.IGNORECASE)
safe = _strip_check(safe).strip().rstrip(",")
try:
conn.execute(f"ALTER TABLE {table} ADD COLUMN IF NOT EXISTS {safe}")
except Exception:
# rollback done by the shim; the next column starts a fresh tx
pass
def _fix_identity_sequences(conn):
"""Sync every IDENTITY sequence to max(id) (safe after explicit id inserts)."""
try:
rows = conn.execute(
"SELECT table_name, column_name FROM information_schema.columns "
"WHERE table_schema = current_schema() AND is_identity = 'YES'"
).fetchall()
except Exception:
conn._safe_rollback()
return
for r in rows:
t, c = r["table_name"], r["column_name"]
try:
conn.execute(
f"SELECT setval(pg_get_serial_sequence('{t}', '{c}'), "
f"GREATEST((SELECT COALESCE(MAX({c}), 1) FROM {t}), 1))"
)
except Exception:
conn._safe_rollback()
# Timestamp columns whose DEFAULT was generated by early builds with the wrong
# to_char pattern ('YYYY-MM-DD HH:MM:SS' — HH is 12-hour, MM is the month).
_TIMESTAMP_DEFAULT_FIX = [
("auth_tokens", "created_at"),
("users", "created_at"),
("calendar_groups", "created_at"),
("weather_cache", "updated_at"),
("weather_hourly_cache", "fetched_at"),
]
def _fix_timestamp_defaults(conn):
"""Repair legacy DEFAULT expressions (idempotent, best-effort)."""
for t, c in _TIMESTAMP_DEFAULT_FIX:
try:
conn.execute(
f"ALTER TABLE {t} ALTER COLUMN {c} "
"SET DEFAULT to_char(now(), 'YYYY-MM-DD HH24:MI:SS')"
)
except Exception:
conn._safe_rollback()
def init_db(db_path: str = "data.db"):
"""Initialize the database with schema-version aware migration (PostgreSQL).
- Fresh database (no `meta` row) -> create the latest SCHEMA + defaults.
- Existing database:
* DB version == DB_VERSION -> use as-is (fast path, defaults ensured)
* no version / DB version < DB_VERSION
-> create missing tables/columns; missing
columns are filled with their SCHEMA
DEFAULT; stamp the new version
* DB version > DB_VERSION -> abort with an error (DB is newer than
this program)
"""
conn = get_db(db_path)
try:
old_version = _read_db_version(conn)
if old_version is not None and old_version > DB_VERSION:
raise SystemExit(
f"[FATAL] 数据库 schema 版本 (v{old_version}) 高于当前程序支持的版本 "
f"(v{DB_VERSION})。请先升级程序,或删除/降级数据库后再启动。"
)
if old_version is not None and old_version == DB_VERSION:
# Same version — no upgrade needed. Just ensure defaults exist.
_ensure_defaults(conn)
_fix_timestamp_defaults(conn)
conn.commit()
return
# Create any missing tables (CREATE TABLE IF NOT EXISTS is idempotent),
# then add any columns missing from older databases. The DDL is
# committed first so a later best-effort column migration (which may
# roll back its own transaction) can never undo the schema creation.
conn.executescript(SCHEMA)
conn.commit()
_migrate_missing_columns(conn)
# Bootstrap defaults (Root user, meta row, config rows).
_ensure_defaults(conn)
# Repair legacy timestamp DEFAULT expressions (old to_char HH:MM pattern).
_fix_timestamp_defaults(conn)
# Stamp the database with the program's schema version.
conn.execute("UPDATE meta SET version=? WHERE id=1", (DB_VERSION,))
conn.execute("INSERT OR IGNORE INTO meta (id, version) VALUES (1, ?)", (DB_VERSION,))
_fix_identity_sequences(conn)
conn.commit()
finally:
conn.close()
+252
View File
@@ -0,0 +1,252 @@
"""PostgreSQL compatibility layer for the legacy sqlite3-style data access.
The rest of the codebase talks to the database through hand-written SQL that
was originally written for sqlite3 (``?`` placeholders, ``datetime('now')``,
``INSERT OR IGNORE``, ``cursor.lastrowid``, ``conn.total_changes`` ...).
This module exposes a sqlite3-like connection/cursor interface on top of
psycopg2 and transparently translates the dialect differences, so the
business-layer SQL keeps working with minimal changes.
Translations applied inside :meth:`PgCursor.execute`:
- ``?`` placeholders -> psycopg2 ``%s`` placeholders
- ``datetime('now')`` -> ``to_char(now(), 'YYYY-MM-DD HH24:MI:SS')``
(UTC, byte-identical format to before)
- ``INSERT OR IGNORE INTO`` -> ``INSERT INTO ... ON CONFLICT DO NOTHING``
- ``BEGIN IMMEDIATE`` -> ``BEGIN``
- ``INSERT`` (without RETURNING) -> appends ``RETURNING <pk>`` so that
``cursor.lastrowid`` keeps working; the
primary key is looked up once from
information_schema and cached.
Rows are returned as a dict subclass that supports both ``row["col"]`` and
``row[0]`` (positional) access to match the old ``sqlite3.Row`` behaviour.
The connection timezone is pinned to UTC so ``now()`` output matches the
previous ``datetime('now')`` (UTC) semantics.
"""
import re
import threading
import psycopg2
import psycopg2.extras
_DATETIME_NOW_RE = re.compile(r"datetime\('now'\)", re.IGNORECASE)
_INSERT_OR_IGNORE_RE = re.compile(r"INSERT\s+OR\s+IGNORE\s+INTO", re.IGNORECASE)
_INSERT_TABLE_RE = re.compile(
r"^\s*INSERT\s+(?:OR\s+IGNORE\s+)?INTO\s+([\"'`\w]+)", re.IGNORECASE
)
_BEGIN_IMMEDIATE_RE = re.compile(r"^\s*BEGIN\s+IMMEDIATE", re.IGNORECASE)
_PK_CACHE: dict[str, str | None] = {}
_PK_LOCK = threading.Lock()
class _Row(dict):
"""dict-like row supporting both column-name and positional access."""
__slots__ = ("_cols",)
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._cols = None
def __getitem__(self, key):
if isinstance(key, int):
if self._cols is None or key < 0 or key >= len(self._cols):
raise IndexError(key)
return dict.__getitem__(self, self._cols[key])
return dict.__getitem__(self, key)
def get_by_index(self, i):
return self[i]
def _wrap_row(row):
"""Convert a psycopg2 result row into a `_Row` (positional access enabled)."""
if row is None or isinstance(row, _Row):
return row
if isinstance(row, dict):
r = _Row(row)
r._cols = list(row.keys())
return r
return row
def _primary_key(conn, table: str) -> str | None:
"""Return the primary-key column of `table` (cached per process)."""
with _PK_LOCK:
if table in _PK_CACHE:
return _PK_CACHE[table]
pk: str | None = None
try:
cur = conn.cursor()
cur.execute(
"""
SELECT kcu.column_name
FROM information_schema.table_constraints tc
JOIN information_schema.key_column_usage kcu
ON tc.constraint_name = kcu.constraint_name
AND tc.table_schema = kcu.table_schema
WHERE tc.constraint_type = 'PRIMARY KEY'
AND tc.table_name = %s
AND tc.table_schema = current_schema()
ORDER BY kcu.ordinal_position
LIMIT 1
""",
(table,),
)
row = cur.fetchone()
pk = row["column_name"] if row else None
cur.close()
except Exception:
pk = None
with _PK_LOCK:
_PK_CACHE[table] = pk
return pk
def _translate_sql(sql: str) -> tuple[str, bool]:
"""Return (translated_sql, has_insert_or_ignore)."""
sql = _DATETIME_NOW_RE.sub("to_char(now(), 'YYYY-MM-DD HH24:MI:SS')", sql)
sql = _BEGIN_IMMEDIATE_RE.sub("BEGIN", sql)
has_ignore = bool(_INSERT_OR_IGNORE_RE.search(sql))
if has_ignore:
sql = _INSERT_OR_IGNORE_RE.sub("INSERT INTO", sql)
# `?` -> `%s`. The codebase SQL contains no literal `?` inside strings.
sql = sql.replace("?", "%s")
return sql, has_ignore
class PgCursor:
"""A psycopg2 cursor wrapper exposing the sqlite3 cursor surface."""
def __init__(self, proxy: "PgConnection", cursor):
self._proxy = proxy
self._cursor = cursor
self._lastrowid = None
def execute(self, sql: str, params=None) -> "PgCursor":
sql, has_ignore = _translate_sql(sql)
conn = self._proxy._conn
m = _INSERT_TABLE_RE.match(sql)
table = m.group(1).strip('"`') if m else None
returning_pk = None
if table is not None and " RETURNING " not in sql.upper():
returning_pk = _primary_key(conn, table)
sql = sql.rstrip().rstrip(";")
if has_ignore:
sql += " ON CONFLICT DO NOTHING"
if returning_pk:
sql += f" RETURNING {returning_pk}"
try:
self._cursor.execute(sql, params if params is not None else ())
if returning_pk:
row = _wrap_row(self._cursor.fetchone())
self._lastrowid = row[0] if row is not None else None
rc = self._cursor.rowcount
if rc and rc > 0:
self._proxy._total_changes += rc
except Exception:
self._proxy._safe_rollback()
raise
return self
# ── sqlite3 cursor surface ───────────────────────────────
@property
def lastrowid(self):
return self._lastrowid
@property
def rowcount(self):
return self._cursor.rowcount
def fetchone(self):
return _wrap_row(self._cursor.fetchone())
def fetchall(self):
return [_wrap_row(r) for r in self._cursor.fetchall()]
def fetchmany(self, size=None):
return [_wrap_row(r) for r in self._cursor.fetchmany(size)]
def close(self):
try:
self._cursor.close()
except Exception:
pass
class PgConnection:
"""A sqlite3-like connection backed by psycopg2.
Note: unlike a raw psycopg2 connection, a failed statement automatically
rolls the connection back (sqlite allows continuing after an error; this
keeps the connection usable under the same call pattern).
"""
def __init__(self, dsn: str):
self._conn = psycopg2.connect(dsn, cursor_factory=psycopg2.extras.RealDictCursor)
self._conn.autocommit = False
self._total_changes = 0
self._closed = False
cur = self._conn.cursor()
cur.execute("SET TIME ZONE 'UTC'")
cur.close()
def execute(self, sql: str, params=None) -> PgCursor:
return self.cursor().execute(sql, params)
def cursor(self) -> PgCursor:
return PgCursor(self, self._conn.cursor())
def executescript(self, sql: str):
"""Run a multi-statement DDL script (SCHEMA). No translation."""
cur = self._conn.cursor()
try:
cur.execute(sql)
except Exception:
self._safe_rollback()
raise
finally:
try:
cur.close()
except Exception:
pass
def commit(self):
self._conn.commit()
def rollback(self):
self._conn.rollback()
def close(self):
if not self._closed:
try:
self._conn.rollback()
self._conn.close()
except Exception:
pass
self._closed = True
def _safe_rollback(self):
try:
self._conn.rollback()
except Exception:
pass
# ── sqlite3 connection surface ───────────────────────────
@property
def total_changes(self):
return self._total_changes
@property
def closed(self):
return self._closed
def connect(dsn: str) -> PgConnection:
"""Open a PostgreSQL connection with the sqlite3-compatible wrapper."""
return PgConnection(dsn)
-3
View File
@@ -2,10 +2,7 @@
State is stored per user (one row per user_id).
"""
import json
import os
import time
from datetime import datetime, timedelta
from backend.time_util import now as tz_now
from flask import current_app
from backend.models import get_db
+118 -68
View File
@@ -8,13 +8,11 @@ Multi-user rules enforced here:
the MFA setup page until they configure (and verify) one real login method,
at which point the temp TOTP is invalidated.
"""
import json
import os
import queue
import io
import time as _time
import secrets
from datetime import datetime
from flask import (
Blueprint, request, jsonify, Response,
render_template, current_app, stream_with_context,
@@ -23,15 +21,15 @@ from flask import (
from backend import calendar_mgr, weather_mgr, alarm_mgr, pomodoro_mgr, countdown_mgr
from backend.auth import (
login_required, login_or_temp_required, display_login_required, admin_required,
is_authenticated, is_temp_pending, is_admin, current_user_id,
is_authenticated, is_temp_pending, current_user_id,
verify_login, verify_totp, verify_recovery_code,
check_rate_limit, record_failure, record_success,
require_reauth, mark_reauth, generate_recovery_codes, generate_temp_password,
require_reauth, mark_reauth, generate_recovery_codes,
load_auth_context, issue_tokens, set_refresh_cookie, clear_refresh_cookie,
rotate_refresh, logout_current_device, load_page_context, REFRESH_COOKIE,
)
from backend.models import (
get_auth_config, update_auth_config, set_temp_password, clear_temp_password,
get_auth_config, update_auth_config, clear_temp_password,
set_recovery_codes, get_recovery_codes, has_mfa_configured,
get_display_config, set_display_mode, set_custom_html,
set_custom_html_template, apply_custom_html,
@@ -39,6 +37,10 @@ from backend.models import (
is_temp_account, is_admin_user, finalize_temp_account,
find_user_by_credential_id, bump_all_session_versions,
bump_user_session_version, revoke_user_tokens, revoke_all_tokens,
get_db, _read_db_version,
)
from backend.db_admin import (
export_database_json, import_blob, detect_format,
)
from backend.webauthn_mgr import (
begin_registration, complete_registration,
@@ -64,6 +66,14 @@ def _load_token():
load_auth_context()
@api.before_request
def _adjust_upload_limit():
"""Database import accepts large legacy .db files; raise this request's own
body limit (per-request setter, no global race) instead of the global 5MB cap."""
if request.path == "/api/admin/db/import" and request.method == "POST":
request.max_content_length = 200 * 1024 * 1024
@api.before_request
def _check_origin():
"""Protect state-changing requests against CSRF by verifying Origin/Referer.
@@ -107,12 +117,6 @@ def login_page():
return render_template("login.html", next=next_url)
@api.route("/register-request")
def register_request_page():
"""Public page where a prospective user fills in their info (admin approves later)."""
return render_template("register_request.html")
@api.route("/passkey")
def passkey_page():
"""Passkey bridge page: loaded inside the ASWebAuthenticationSession sheet
@@ -492,39 +496,6 @@ def api_auth_status():
})
# ════════════════════════════════════════════
# USER SELF-SERVICE REQUEST
# ════════════════════════════════════════════
# The user fills in their info; the admin confirms it and a temp account
# (with a server-held temp TOTP) is generated automatically.
_REG_REQ_LIMIT: dict[str, list] = {} # ip -> [timestamps]
@api.route("/api/auth/register-request", methods=["POST"])
def api_register_request():
"""Public: a prospective user submits their remark for admin approval.
Rate-limited per IP (5 requests per 10 minutes)."""
ip = request.remote_addr or "unknown"
now = _time.time()
stamps = [t for t in _REG_REQ_LIMIT.get(ip, []) if now - t < 600]
if len(stamps) >= 5:
_REG_REQ_LIMIT[ip] = stamps
return jsonify({"error": "申请过于频繁,请稍后再试"}), 429
stamps.append(now)
_REG_REQ_LIMIT[ip] = stamps
data = request.get_json() or {}
remark = (data.get("remark") or "").strip()
if not remark:
return jsonify({"error": "请填写你的称呼/备注"}), 400
if len(remark) > 64:
return jsonify({"error": "备注过长"}), 400
create_user(remark=remark, role="user", is_temp=0, status="pending")
return jsonify({"ok": True, "message": "已提交申请,请等待管理员确认"}), 201
# ════════════════════════════════════════════
# MFA CONFIGURATION
# ════════════════════════════════════════════
@@ -842,30 +813,6 @@ def admin_create_user():
return jsonify({"ok": True, "user": _user_payload(uid)}), 201
@api.route("/api/admin/users/confirm", methods=["POST"])
@admin_required
def admin_confirm_request():
"""Confirm a pending self-service request → temp account with temp TOTP."""
if not require_reauth():
return jsonify({"error": "需要重新验证身份", "reauth_required": True}), 403
data = request.get_json() or {}
request_id = data.get("request_id")
if request_id is None:
return jsonify({"error": "request_id required"}), 400
u = get_user(request_id)
if not u:
return jsonify({"error": "申请不存在"}), 404
if u.get("status") != "pending":
return jsonify({"error": "该申请已处理"}), 400
import pyotp
secret = pyotp.random_base32()
update_user(request_id, is_temp=1, temp_totp_secret=secret, status="active")
return jsonify({"ok": True, "user": _user_payload(request_id)})
@api.route("/api/admin/users/<int:user_id>", methods=["PUT"])
@admin_required
def admin_update_user(user_id):
@@ -1052,6 +999,109 @@ def admin_kick_all_users():
return jsonify({"ok": True})
# ════════════════════════════════════════════
# DATABASE ADMINISTRATION (Root)
# ════════════════════════════════════════════
# Importing an old SQLite database, exporting a backup, or importing a backup
# replaces ALL current data — Root-only and re-auth required.
def _require_root():
"""Database administration is Root-only (id 0)."""
if _cur() != 0:
return jsonify({"error": "仅 Root 用户可执行数据库管理操作"}), 403
return None
@api.route("/api/admin/db/status", methods=["GET"])
@admin_required
def admin_db_status():
denied = _require_root()
if denied:
return denied
db_path = current_app.config.get("DB_PATH", "data.db")
conn = get_db(db_path)
try:
version = _read_db_version(conn) or 0
server = conn.execute("SELECT version()").fetchone()[0]
rows = conn.execute(
"SELECT table_name FROM information_schema.tables "
"WHERE table_schema = current_schema() ORDER BY table_name"
).fetchall()
tables = [r["table_name"] for r in rows]
finally:
conn.close()
return jsonify({
"database": "postgresql",
"db_version": version,
"server": server,
"tables": tables,
"backup_format": "iot-dboard-backup",
})
@api.route("/api/admin/db/export", methods=["GET"])
@admin_required
def admin_db_export():
denied = _require_root()
if denied:
return denied
if not require_reauth():
return jsonify({"error": "需要重新验证身份", "reauth_required": True}), 403
db_path = current_app.config.get("DB_PATH", "data.db")
try:
payload = export_database_json(db_path)
except Exception as e:
return jsonify({"error": f"导出失败: {e}"}), 500
from datetime import datetime
fname = f"iot-dboard-backup-{datetime.now().strftime('%Y%m%d-%H%M%S')}.json"
return Response(
payload,
mimetype="application/json",
headers={"Content-Disposition": f'attachment; filename="{fname}"'},
)
@api.route("/api/admin/db/import", methods=["POST"])
@admin_required
def admin_db_import():
denied = _require_root()
if denied:
return denied
if not require_reauth():
return jsonify({"error": "需要重新验证身份", "reauth_required": True}), 403
file = request.files.get("file")
if not file or not file.filename:
return jsonify({"error": "请选择要导入的数据库文件"}), 400
data = file.read()
if not data:
return jsonify({"error": "文件为空"}), 400
if len(data) > 200 * 1024 * 1024:
return jsonify({"error": "文件超过 200MB 限制"}), 413
db_path = current_app.config.get("DB_PATH", "data.db")
fmt = detect_format(data)
try:
stats = import_blob(db_path, data)
except ValueError as e:
return jsonify({"error": str(e)}), 400
except Exception as e:
return jsonify({"error": f"导入失败(已回滚,当前数据未受影响): {e}"}), 500
total = sum(v for k, v in (stats or {}).items() if isinstance(v, int))
skipped = (stats or {}).get("_skipped_tables") or []
msg = f"导入成功:{total} 行数据已恢复。所有设备已下线,请重新登录。"
if skipped:
msg += f" 以下表因结构与当前版本不兼容被跳过:{', '.join(skipped)}。"
return jsonify({
"ok": True,
"format": fmt,
"stats": stats,
"message": msg,
"session_invalidated": True,
})
# ════════════════════════════════════════════
# DISPLAY PAGES
# ════════════════════════════════════════════
+129 -2
View File
@@ -4,6 +4,8 @@ All user-scoped jobs iterate over every active user (with that user's
timezone/context active) so each user's alarms, reminders, calendars,
pomodoro, countdowns and weather are serviced independently.
"""
import threading
from apscheduler.schedulers.background import BackgroundScheduler
from backend.calendar_mgr import sync_all_subscriptions, get_upcoming_reminders
from backend.weather_mgr import fetch_weather, fetch_hourly
@@ -16,6 +18,129 @@ from backend.time_util import set_thread_user, clear_thread_user
_scheduler: BackgroundScheduler | None = None
# ─── Scheduler single-instance guard ─────────
# With Gunicorn every worker imports the app; the background scheduler must run
# in exactly ONE process. A PostgreSQL session-level advisory lock decides which
# worker owns it (the lock is held by an open connection and released when that
# process exits, at which point another worker takes over).
_SCHEDULER_LOCK_KEY = 0x494F5444424F4152 # "IOTDBOAR"
_scheduler_lock_conn = None
_watchdog: "threading.Thread | None" = None
_WATCHDOG_INTERVAL = 30 # seconds
def _lock_connect(db_url: str):
"""Open the lock connection with TCP keepalive + connect timeout.
Without keepalives a network blip could leave PostgreSQL holding the lock
for ~2h (default tcp keepalive), blocking every worker from taking over.
(libpq URI query params do NOT accept tcp_keepalives_* — pass them as
psycopg2 keyword args instead.)"""
import psycopg2
return psycopg2.connect(
db_url,
connect_timeout=5,
keepalives_idle=30,
keepalives_interval=10,
keepalives_count=3,
)
def try_acquire_scheduler_lock(db_url: str) -> bool:
"""Try to become the scheduler owner. Returns True for the winner."""
global _scheduler_lock_conn
conn = None
try:
conn = _lock_connect(db_url)
cur = conn.cursor()
cur.execute("SELECT pg_try_advisory_lock(%s)", (_SCHEDULER_LOCK_KEY,))
ok = bool(cur.fetchone()[0])
cur.close()
if ok:
_scheduler_lock_conn = conn
return True
conn.close()
return False
except Exception:
try:
if _scheduler_lock_conn:
_scheduler_lock_conn.close()
_scheduler_lock_conn = None
except Exception:
pass
try:
if conn is not None:
conn.close() # close the partially-used local connection
except Exception:
pass
return False
def release_scheduler_lock():
"""Release the advisory lock (also closes the holding connection)."""
global _scheduler_lock_conn
if _scheduler_lock_conn:
try:
_scheduler_lock_conn.close()
except Exception:
pass
_scheduler_lock_conn = None
def _watchdog_loop(app, db_url: str):
"""Keeps the scheduler alive across worker restarts/freezes:
- lock owner: periodically checks the lock connection is healthy;
- non-owner: periodically retries to take over the lock.
"""
global _scheduler
import time
while True:
time.sleep(_WATCHDOG_INTERVAL)
try:
if _scheduler_lock_conn is not None:
# Owner: health check on the lock connection, and make sure the
# scheduler thread itself is still running (start failure or a
# dead thread must not hold the lock forever).
cur = _scheduler_lock_conn.cursor()
cur.execute("SELECT 1")
cur.fetchone()
cur.close()
if _scheduler is None or not _scheduler.running:
if _scheduler:
try:
stop_scheduler()
except Exception:
pass
start_scheduler(app)
app.logger.info("Scheduler restarted (lock held but not running)")
elif _scheduler is None:
# Non-owner: try to take over (previous owner died/froze).
if try_acquire_scheduler_lock(db_url):
start_scheduler(app)
app.logger.info("Scheduler started (watchdog takeover)")
except Exception:
# Lock connection became unhealthy → drop the lock and re-race.
release_scheduler_lock()
if _scheduler:
try:
stop_scheduler()
except Exception:
pass
try:
if try_acquire_scheduler_lock(db_url):
start_scheduler(app)
app.logger.info("Scheduler re-acquired lock after watchdog failure")
except Exception:
pass
def start_watchdog(app, db_url: str):
"""Start the scheduler watchdog thread (idempotent)."""
global _watchdog
if _watchdog is not None and _watchdog.is_alive():
return
_watchdog = threading.Thread(target=_watchdog_loop, args=(app, db_url), daemon=True)
_watchdog.start()
def _each_user(app, fn):
"""Run fn(user_id) for every active user inside the app context."""
@@ -82,5 +207,7 @@ def start_scheduler(app):
def stop_scheduler():
global _scheduler
if _scheduler:
_scheduler.shutdown(wait=False)
_scheduler = None
try:
_scheduler.shutdown(wait=False)
finally:
_scheduler = None
+2 -3
View File
@@ -15,7 +15,6 @@ The timezone is cached per-request via Flask app config to avoid DB lookups.
import time as _time
import threading
from datetime import datetime, timedelta, timezone, date
from typing import Union
import pytz
@@ -130,14 +129,14 @@ def set_tz(tz_str: str) -> bool:
# Convert to an IANA-like name: pytz has FixedOffset but easiest is
# just use a fixed-offset timezone
import pytz
tz = pytz.FixedOffset(sign * hours * 60) # FixedOffset expects minutes
pytz.FixedOffset(sign * hours * 60) # FixedOffset expects minutes
except (ValueError, IndexError):
return False
else:
# IANA name — validate by trying to load
try:
import pytz
tz = pytz.timezone(normalized)
pytz.timezone(normalized)
except Exception:
return False
+2 -4
View File
@@ -2,7 +2,7 @@
import json
import requests
from datetime import datetime
from backend.time_util import now as tz_now, strftime as tz_strftime, today_str as tz_today_str, strptime as tz_strptime, localize as tz_localize
from backend.time_util import now as tz_now, strftime as tz_strftime, strptime as tz_strptime, localize as tz_localize
from flask import current_app
from backend.models import get_db
from backend.jwt_util import sign_jwt, generate_key_pair
@@ -353,10 +353,8 @@ def get_next_update_times(user_id: int = None) -> dict:
h_updated = None
h_row = db.execute("SELECT data FROM weather_hourly_cache WHERE user_id=?", (user_id,)).fetchone()
if h_row and h_row["data"]:
h_data = json.loads(h_row["data"])
# Hourly data is a list; we store fetched_at separately, but for display
# use the SQLite fetched_at (converted) or just show the schedule
# Let's use the local-computed schedule as last-update signal
# use the local-computed schedule as last-update signal
h_row2 = db.execute("SELECT fetched_at FROM weather_hourly_cache WHERE user_id=?", (user_id,)).fetchone()
if h_row2 and h_row2[0]:
# Keep the UTC string but note it's UTC — we'll just pass it
-2
View File
@@ -108,7 +108,6 @@ def begin_registration(request, user_id: int, display_name: str = "",
existing_credential_id: str = None) -> dict:
"""Start WebAuthn registration for a specific user. Returns options dict."""
rp_id = _get_rp_id(request)
origin = _get_origin(request)
user_name = f"user_{user_id}"
user_handle = user_name.encode("utf-8")
@@ -200,7 +199,6 @@ def begin_authentication(request, allow_credential_ids: list = None) -> dict:
ceremony to (used for re-authentication of the current user).
"""
rp_id = _get_rp_id(request)
origin = _get_origin(request)
allow_credentials = []
if allow_credential_ids: