From d79ba5d2081f07a557848a6482d7be10a56050aa Mon Sep 17 00:00:00 2001 From: Linus Vogel Date: Wed, 24 Dec 2025 12:18:31 +0100 Subject: [PATCH] database auto migrations should work now --- alembic.ini | 148 ++++++++++++++++++ pillar_tool.toml.example | 2 +- pillar_tool/__init__.py | 9 +- .../__pycache__/__init__.cpython-313.pyc | Bin 1900 -> 2355 bytes pillar_tool/db/__init__.py | 4 +- pillar_tool/db/database.py | 2 +- .../db/migrations}/README | 0 pillar_tool/db/migrations/__init__.py | 0 .../db/migrations}/env.py | 27 ++-- .../db/migrations}/script.py.mako | 0 pillar_tool/db/models/user.py | 5 +- pillar_tool/util/__init__.py | 10 +- 12 files changed, 182 insertions(+), 25 deletions(-) create mode 100644 alembic.ini rename {migrations => pillar_tool/db/migrations}/README (100%) create mode 100644 pillar_tool/db/migrations/__init__.py rename {migrations => pillar_tool/db/migrations}/env.py (87%) rename {migrations => pillar_tool/db/migrations}/script.py.mako (100%) diff --git a/alembic.ini b/alembic.ini new file mode 100644 index 0000000..e0a6d6d --- /dev/null +++ b/alembic.ini @@ -0,0 +1,148 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts. +# this is typically a path given in POSIX (e.g. forward slashes) +# format, relative to the token %(here)s which refers to the location of this +# ini file +script_location = %(here)s/pillar_tool/db/migrations + +# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s +# Uncomment the line below if you want the files to be prepended with date and time +# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file +# for all available tokens +# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s + +# sys.path path, will be prepended to sys.path if present. +# defaults to the current working directory. for multiple paths, the path separator +# is defined by "path_separator" below. +prepend_sys_path = . + + +# timezone to use when rendering the date within the migration file +# as well as the filename. +# If specified, requires the tzdata library which can be installed by adding +# `alembic[tz]` to the pip requirements. +# string value is passed to ZoneInfo() +# leave blank for localtime +# timezone = + +# max length of characters to apply to the "slug" field +# truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; This defaults +# to /versions. When using multiple version +# directories, initial revisions must be specified with --version-path. +# The path separator used here should be the separator specified by "path_separator" +# below. +# version_locations = %(here)s/bar:%(here)s/bat:%(here)s/alembic/versions + +# path_separator; This indicates what character is used to split lists of file +# paths, including version_locations and prepend_sys_path within configparser +# files such as alembic.ini. +# The default rendered in new alembic.ini files is "os", which uses os.pathsep +# to provide os-dependent path splitting. +# +# Note that in order to support legacy alembic.ini files, this default does NOT +# take place if path_separator is not present in alembic.ini. If this +# option is omitted entirely, fallback logic is as follows: +# +# 1. Parsing of the version_locations option falls back to using the legacy +# "version_path_separator" key, which if absent then falls back to the legacy +# behavior of splitting on spaces and/or commas. +# 2. Parsing of the prepend_sys_path option falls back to the legacy +# behavior of splitting on spaces, commas, or colons. +# +# Valid values for path_separator are: +# +# path_separator = : +# path_separator = ; +# path_separator = space +# path_separator = newline +# +# Use os.pathsep. Default configuration used for new projects. +path_separator = os + +# set to 'true' to search source files recursively +# in each "version_locations" directory +# new in Alembic version 1.10 +# recursive_version_locations = false + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + +# database URL. This is consumed by the user-maintained env.py script only. +# other means of configuring database URLs may be customized within the env.py +# file. +# Note: this is overridden in the env.py file and not necessary here +#sqlalchemy.url = driver://user:pass@localhost/dbname + + +[post_write_hooks] +# post_write_hooks defines scripts or Python functions that are run +# on newly generated revision scripts. See the documentation for further +# detail and examples + +# format using "black" - use the console_scripts runner, against the "black" entrypoint +# hooks = black +# black.type = console_scripts +# black.entrypoint = black +# black.options = -l 79 REVISION_SCRIPT_FILENAME + +# lint with attempts to fix using "ruff" - use the module runner, against the "ruff" module +# hooks = ruff +# ruff.type = module +# ruff.module = ruff +# ruff.options = check --fix REVISION_SCRIPT_FILENAME + +# Alternatively, use the exec runner to execute a binary found on your PATH +# hooks = ruff +# ruff.type = exec +# ruff.executable = ruff +# ruff.options = check --fix REVISION_SCRIPT_FILENAME + +# Logging configuration. This is also consumed by the user-maintained +# env.py script only. +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARNING +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARNING +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/pillar_tool.toml.example b/pillar_tool.toml.example index f6d8919..d57843c 100644 --- a/pillar_tool.toml.example +++ b/pillar_tool.toml.example @@ -2,5 +2,5 @@ user = "db user" password = "db password" host = "db host" -post = 5432 +port = 5432 database = "db name" \ No newline at end of file diff --git a/pillar_tool/__init__.py b/pillar_tool/__init__.py index 563ef3f..9acf1d9 100644 --- a/pillar_tool/__init__.py +++ b/pillar_tool/__init__.py @@ -1,3 +1,8 @@ + +# load config so everything else can work +from pillar_tool.util import load_config, config +load_config() + from contextlib import asynccontextmanager from fastapi import FastAPI @@ -7,12 +12,8 @@ from fastapi.responses import HTMLResponse, PlainTextResponse from starlette.responses import JSONResponse from pillar_tool.middleware.basicauth_backend import BasicAuthBackend -from pillar_tool.util import load_config, config from pillar_tool.db import run_db_migrations -# load config so everything else can work -load_config() - @asynccontextmanager async def app_lifespan(app: FastAPI): diff --git a/pillar_tool/__pycache__/__init__.cpython-313.pyc b/pillar_tool/__pycache__/__init__.cpython-313.pyc index 6908bf3e34786b51474a493f8012e4958c6b2f97..42b00d62af72ea8aed9aea6f929d056ac042bc5f 100644 GIT binary patch delta 1399 zcmZ8f&2Jl35PxsK{1rQ1f5eVMn#HM5w^WJJHkg!xs6++WQVv=5fVz^|*c)fjX4jov zqar{?g(DKFR*`z*#(`TUj{E^VaG|u9t`u=;4n;^9`A~6V-mYyDR@!;NXY(Fiv!W5wuYqQzFD;T?Bod(9bR+_=f`ZD&raHijLy^30W z!JsM^W^C8!wmYt~XZ1oUK2;hP4ev9jgIC|Y>mL}7(K8{qLvK+vctlr+KhjW!fz$5p zwhycx7{28?UdS8$erM8FR0)TYM=`#L$|{n1L~hUg)2T3y8p5aKNK{CVH26KL(rw7H z@+i#X;jt%5czb={Juug8%NYc}a66ahFk9QU_DrvDI8Zv87Vufnxx^6!#+#WLMiSHk=xuJ5|`+Q7GLU=s(%PMDsonr-a@>mp3- z8J=(SEs%omM75D%wgOtfHhte*1M|+n^n4GJ%pp&}Xr6EhDYQd6_)A=*nII*ty*bB# zxzi3OgBMs ziwo`-wxbPB-J*o-5|fy;Ldq1VE^MhHl^L+uW)-=tfD+qsB~eykr+RJ+UDZ;|q?MVL z&8`A@AcNL{bjL{6qD4{?^-v&Z<(5WUcBi(Ol|E%znwgrx!pObU$!ghFriAUHY>^_d zvTcXN^syGDQ=9333^P+WZKMyA|BtPmu(MYGv}ESQ5GU;qRC|ox1CidK5&uP(=pLVw zMtPOJnNSgpSP(*_t~>Us1Hm4H6oQZzkt8^Q@6`h*VCd>P|12%{WxV>by&0}AI{+TQ z07km`UHR3VhL*u1#Rs<@f=CTLLY+a?PM||SI#CW&RGqp5cCDSIfN+kNl%kwRi&vHV z1sx3#Uh9nvo)^Lh`h)ztQryZ;_-?Ic!^C6X27BWP))n_kYtKkSk@0NfY3is(aoxM* z-bvh~cgR(0N;C8#xz5(8!4{#2JlvAH;?+Y3he*bL3WmbkWF1>Y5#dw(kPtY1Jr*P}GYE)SNJM#=&{G z=>%Z_3cs!96A281%4_Ob+Qpx%&6$Ns42Xe(_o$8-rL;kY8f2kCE`KFA8|2DQGJlldFOr6|?eY)F hN%oD({5V;>Q20cLjwDK_U$Q2_tI3Eo{96EVp}z`|xJ3W} diff --git a/pillar_tool/db/__init__.py b/pillar_tool/db/__init__.py index fdebf28..708e90c 100644 --- a/pillar_tool/db/__init__.py +++ b/pillar_tool/db/__init__.py @@ -7,7 +7,7 @@ from alembic.command import upgrade from pillar_tool.util import config -from models import * +from .models import * cfg = config() user = cfg.db.user @@ -17,7 +17,7 @@ port = cfg.db.port database = cfg.db.database alembic_cfg = Config() alembic_cfg.set_main_option('script_location', f'{os.path.dirname(os.path.realpath(__file__))}/migrations') -alembic_cfg.set_main_option('sqlalchemy.url', f'postgres://{user}:{password}@{host}:{port}/{database}') +alembic_cfg.set_main_option('sqlalchemy.url', f'postgresql://{user}:{password}@{host}:{port}/{database}') alembic_cfg.set_main_option('prepend_sys_path', '.') diff --git a/pillar_tool/db/database.py b/pillar_tool/db/database.py index ba8004e..3922656 100644 --- a/pillar_tool/db/database.py +++ b/pillar_tool/db/database.py @@ -10,7 +10,7 @@ SessionLocal = sessionmaker( autocommit=False, autoflush=False, bind=create_engine( - url=f"postgresql+psycopg2://{cfg.db.user}:{cfg.db.password}@{cfg.host}:{cfg.port}/{cfg.database}" + url=f"postgresql+psycopg2://{cfg.db.user}:{cfg.db.password}@{cfg.db.host}:{cfg.db.port}/{cfg.db.database}" ) ) diff --git a/migrations/README b/pillar_tool/db/migrations/README similarity index 100% rename from migrations/README rename to pillar_tool/db/migrations/README diff --git a/pillar_tool/db/migrations/__init__.py b/pillar_tool/db/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/migrations/env.py b/pillar_tool/db/migrations/env.py similarity index 87% rename from migrations/env.py rename to pillar_tool/db/migrations/env.py index 2adb551..11c0bc6 100644 --- a/migrations/env.py +++ b/pillar_tool/db/migrations/env.py @@ -6,6 +6,8 @@ from sqlalchemy import pool from alembic import context +from pillar_tool.db.database import Base + # this is the Alembic Config object, which provides # access to the values within the .ini file in use. config = context.config @@ -19,13 +21,22 @@ if config.config_file_name is not None: # for 'autogenerate' support # from myapp import mymodel # target_metadata = mymodel.Base.metadata -target_metadata = None +target_metadata = Base.metadata # other values from the config, defined by the needs of env.py, # can be acquired: # my_important_option = config.get_main_option("my_important_option") # ... etc. +def get_custom_url(): + with open("./pillar_tool.toml", 'rb') as f: + data = tomllib.load(f) + user = data['db']['user'] + password = data['db']['password'] + host = data['db']['host'] + port = data['db']['port'] + database = data['db']['database'] + return f"postgresql://{user}:{password}@{host}:{port}/{database}" def run_migrations_offline() -> None: """Run migrations in 'offline' mode. @@ -39,14 +50,7 @@ def run_migrations_offline() -> None: script output. """ - with open("./pillar_tool.toml", 'rb') as f: - data = tomllib.load(f) - user = data['db']['user'] - password = data['db']['password'] - host = data['db']['host'] - port = data['db']['port'] - database = data['db']['database'] - url = f"postgresql://{user}:{password}@{host}:{port}/{database}" + url = get_custom_url() #url = config.get_main_option("sqlalchemy.url") context.configure( url=url, @@ -66,8 +70,11 @@ def run_migrations_online() -> None: and associate a connection with the context. """ + url = get_custom_url() + cfg = config.get_section(config.config_ini_section, {}) + cfg['sqlalchemy.url'] = url connectable = engine_from_config( - config.get_section(config.config_ini_section, {}), + cfg, prefix="sqlalchemy.", poolclass=pool.NullPool, ) diff --git a/migrations/script.py.mako b/pillar_tool/db/migrations/script.py.mako similarity index 100% rename from migrations/script.py.mako rename to pillar_tool/db/migrations/script.py.mako diff --git a/pillar_tool/db/models/user.py b/pillar_tool/db/models/user.py index fde37e3..d90fddd 100644 --- a/pillar_tool/db/models/user.py +++ b/pillar_tool/db/models/user.py @@ -1,5 +1,4 @@ -from pydantic import UUID4 -from sqlalchemy import Column, String +from sqlalchemy import Column, String, UUID from pillar_tool.db.database import Base @@ -7,7 +6,7 @@ from pillar_tool.db.database import Base class User(Base): __tablename__ = 'users' - id = Column(UUID4, primary_key=True) + id = Column(UUID, primary_key=True) username = Column(String, nullable=False) pw_hash = Column(String, nullable=False) pw_salt = Column(String, nullable=False) diff --git a/pillar_tool/util/__init__.py b/pillar_tool/util/__init__.py index 066349a..2168c0e 100644 --- a/pillar_tool/util/__init__.py +++ b/pillar_tool/util/__init__.py @@ -5,8 +5,8 @@ import tomllib def load_config(): paths: list[str] = [ - "/etc/pillar_tool/config.toml", "./pillar_tool.toml", + "/etc/pillar_tool/config.toml", ] for path in paths: @@ -17,9 +17,11 @@ def load_config(): cfg_dict = tomllib.load(f) cfg = Config(**cfg_dict) globals()['_loaded_config_object'] = cfg - except: - pass - raise ValueError("No valid config file could be found!") + return + except Exception as e: + print(e) + + raise ValueError(f"No valid config file could be found! (working directory: {os.getcwd()})") def config() -> Config: