Add automatic instruction fetcher for official sets
Downloads official instruction PDFs from LEGO, located via the Brickset API, one set per interval (10 minutes by default) so neither API gets hammered. Official sets only; MOCs are excluded throughout. - New admin page at /admin/instruction-fetch: queue builder, pacing controls, live countdown and per-set job history - Database-backed queue (instruction_fetch_jobs) and last-run timestamp, so pacing survives restarts - Background worker thread, plus 'flask instructions queue/run/status' for Task Scheduler or cron - SHA-256 dedupe (instructions.sha256), PDF header check, size cap and streaming downloads via temp files - brickset_api: getInstructions2 by set number and getInstructions by setID (the old code sent a set number to the setID method); getSets params now proper JSON; getKeyUsageStats added - Schema top-ups applied automatically at boot: new table plus instructions.sha256/source - Docker: fetcher env vars passed through compose, no worker thread in the entrypoint bootstrap
This commit is contained in:
jessikitty committed
1 parent
aac2b60517
commit
89f65e2d74
23 files changed
+2073
-76
No files matched your search
@@ -0,0 +1,211 @@
|
||||
@echo off
|
||||
REM ===========================================================================
|
||||
REM LEGO Instructions Manager - Instruction Fetcher installer
|
||||
REM
|
||||
REM RUN THIS FROM THE FOLDER YOU EXTRACTED THE UPDATE INTO, not from your
|
||||
REM LIM install - the two cannot be the same folder.
|
||||
REM
|
||||
REM Copies 22 named files into an existing LIM install, backing up every
|
||||
REM file it replaces first. Whole directories are never copied, so your
|
||||
REM uploads and database are not involved.
|
||||
REM
|
||||
REM Usage: install_instruction_fetcher.bat [target]
|
||||
REM Default target: E:\LIM
|
||||
REM
|
||||
REM Created by: Jess Rogerson (yelling commands at Claude.AI)
|
||||
REM ===========================================================================
|
||||
|
||||
setlocal EnableDelayedExpansion
|
||||
|
||||
set "SOURCE=%~dp0"
|
||||
set "TARGET=%~1"
|
||||
if "%TARGET%"=="" set "TARGET=E:\LIM"
|
||||
set "MISSING=0"
|
||||
set "COPYFAIL=0"
|
||||
set "SAVED=0"
|
||||
|
||||
for %%A in ("%SOURCE%.") do set "SRCFULL=%%~fA"
|
||||
for %%B in ("%TARGET%\.") do set "DSTFULL=%%~fB"
|
||||
|
||||
echo.
|
||||
echo Source : %SRCFULL%
|
||||
echo Target : %DSTFULL%
|
||||
echo.
|
||||
|
||||
REM --- refuse to copy a folder onto itself -----------------------------------
|
||||
if /i "%SRCFULL%"=="%DSTFULL%" goto :samedir
|
||||
|
||||
if not exist "%TARGET%\app\__init__.py" goto :nottarget
|
||||
|
||||
REM --- are the update files actually here? -----------------------------------
|
||||
echo Checking the update files...
|
||||
call :need ".env.docker.example"
|
||||
call :need ".env.example"
|
||||
call :need "README.md"
|
||||
call :need "app\__init__.py"
|
||||
call :need "app\cli.py"
|
||||
call :need "app\config.py"
|
||||
call :need "app\models\__init__.py"
|
||||
call :need "app\models\instruction.py"
|
||||
call :need "app\models\instruction_fetch.py"
|
||||
call :need "app\models\site_setting.py"
|
||||
call :need "app\routes\__init__.py"
|
||||
call :need "app\routes\instruction_fetch.py"
|
||||
call :need "app\services\brickset_api.py"
|
||||
call :need "app\services\fetch_worker.py"
|
||||
call :need "app\services\instruction_fetcher.py"
|
||||
call :need "app\services\schema_upgrade.py"
|
||||
call :need "app\templates\admin\dashboard.html"
|
||||
call :need "app\templates\admin\instruction_fetch.html"
|
||||
call :need "app\templates\admin\settings.html"
|
||||
call :need "docker-compose.yml"
|
||||
call :need "docker\entrypoint.sh"
|
||||
call :need "docs\INSTRUCTION_FETCHER.md"
|
||||
if not "%MISSING%"=="0" goto :incomplete
|
||||
echo all 22 present.
|
||||
|
||||
REM --- backup ------------------------------------------------------------------
|
||||
for /f "tokens=2 delims==" %%I in ('wmic os get localdatetime /value') do set "DT=%%I"
|
||||
set "STAMP=!DT:~0,8!-!DT:~8,6!"
|
||||
set "BACKUP=%TARGET%\_backup_fetcher_!STAMP!"
|
||||
|
||||
echo.
|
||||
echo Backing up the files about to be replaced:
|
||||
echo !BACKUP!
|
||||
echo.
|
||||
mkdir "!BACKUP!" 2>nul
|
||||
call :save ".env.docker.example"
|
||||
call :save ".env.example"
|
||||
call :save "README.md"
|
||||
call :save "app\__init__.py"
|
||||
call :save "app\cli.py"
|
||||
call :save "app\config.py"
|
||||
call :save "app\models\__init__.py"
|
||||
call :save "app\models\instruction.py"
|
||||
call :save "app\models\instruction_fetch.py"
|
||||
call :save "app\models\site_setting.py"
|
||||
call :save "app\routes\__init__.py"
|
||||
call :save "app\routes\instruction_fetch.py"
|
||||
call :save "app\services\brickset_api.py"
|
||||
call :save "app\services\fetch_worker.py"
|
||||
call :save "app\services\instruction_fetcher.py"
|
||||
call :save "app\services\schema_upgrade.py"
|
||||
call :save "app\templates\admin\dashboard.html"
|
||||
call :save "app\templates\admin\instruction_fetch.html"
|
||||
call :save "app\templates\admin\settings.html"
|
||||
call :save "docker-compose.yml"
|
||||
call :save "docker\entrypoint.sh"
|
||||
call :save "docs\INSTRUCTION_FETCHER.md"
|
||||
echo %SAVED% file^(s^) backed up.
|
||||
|
||||
REM --- install -------------------------------------------------------------------
|
||||
echo.
|
||||
echo Installing...
|
||||
call :put ".env.docker.example"
|
||||
call :put ".env.example"
|
||||
call :put "README.md"
|
||||
call :put "app\__init__.py"
|
||||
call :put "app\cli.py"
|
||||
call :put "app\config.py"
|
||||
call :put "app\models\__init__.py"
|
||||
call :put "app\models\instruction.py"
|
||||
call :put "app\models\instruction_fetch.py"
|
||||
call :put "app\models\site_setting.py"
|
||||
call :put "app\routes\__init__.py"
|
||||
call :put "app\routes\instruction_fetch.py"
|
||||
call :put "app\services\brickset_api.py"
|
||||
call :put "app\services\fetch_worker.py"
|
||||
call :put "app\services\instruction_fetcher.py"
|
||||
call :put "app\services\schema_upgrade.py"
|
||||
call :put "app\templates\admin\dashboard.html"
|
||||
call :put "app\templates\admin\instruction_fetch.html"
|
||||
call :put "app\templates\admin\settings.html"
|
||||
call :put "docker-compose.yml"
|
||||
call :put "docker\entrypoint.sh"
|
||||
call :put "docs\INSTRUCTION_FETCHER.md"
|
||||
if not "%COPYFAIL%"=="0" goto :copyfailed
|
||||
|
||||
echo.
|
||||
echo ==========================================================
|
||||
echo Done - 22 files installed.
|
||||
echo.
|
||||
echo No migration step: the new table and columns are created
|
||||
echo the next time LIM starts.
|
||||
echo.
|
||||
echo 1. Restart LIM (start_lim.bat)
|
||||
echo 2. Admin ^> Instruction Fetcher
|
||||
echo 3. Queue sets, then switch the fetcher on
|
||||
echo.
|
||||
echo To push the same change to Gitea, run push_fetcher_update.bat
|
||||
echo FROM THIS FOLDER (not from %TARGET%).
|
||||
echo.
|
||||
echo Rollback: copy the files in
|
||||
echo !BACKUP!
|
||||
echo back over %TARGET%
|
||||
echo ==========================================================
|
||||
echo.
|
||||
pause
|
||||
exit /b 0
|
||||
|
||||
REM --- exits ---------------------------------------------------------------------
|
||||
|
||||
:samedir
|
||||
echo ERROR: the source and target are the same folder.
|
||||
echo.
|
||||
echo You are running this script from inside your LIM install, so there
|
||||
echo is nothing to copy - which is why an earlier run appeared to work
|
||||
echo but changed nothing.
|
||||
echo.
|
||||
echo Open the folder you extracted the update into (the one containing
|
||||
echo app\services\instruction_fetcher.py) and run it from there:
|
||||
echo.
|
||||
echo cd /d C:\path\to\lim-instruction-fetcher
|
||||
echo install_instruction_fetcher.bat %TARGET%
|
||||
goto :fail
|
||||
|
||||
:nottarget
|
||||
echo ERROR: %TARGET% does not look like a LIM install
|
||||
echo ^(no app\__init__.py^).
|
||||
echo.
|
||||
echo Pass the right path, e.g. install_instruction_fetcher.bat D:\LIM
|
||||
goto :fail
|
||||
|
||||
:incomplete
|
||||
echo.
|
||||
echo ERROR: %MISSING% file^(s^) missing from
|
||||
echo %SRCFULL%
|
||||
echo.
|
||||
echo This is not the extracted update folder. Look for the one
|
||||
echo containing app\services\instruction_fetcher.py.
|
||||
goto :fail
|
||||
|
||||
:copyfailed
|
||||
echo.
|
||||
echo ERROR: %COPYFAIL% file^(s^) could not be copied. Is LIM still running?
|
||||
echo Stop it and run this again; the backup in
|
||||
echo !BACKUP!
|
||||
echo has the originals.
|
||||
goto :fail
|
||||
|
||||
:need
|
||||
if not exist "%SOURCE%%~1" echo MISSING: %~1 & set /a MISSING+=1
|
||||
goto :eof
|
||||
|
||||
:save
|
||||
if not exist "%TARGET%\%~1" goto :eof
|
||||
for %%F in ("!BACKUP!\%~1") do if not exist "%%~dpF" mkdir "%%~dpF"
|
||||
copy /Y "%TARGET%\%~1" "!BACKUP!\%~1" >nul
|
||||
if not errorlevel 1 echo saved %~1 & set /a SAVED+=1
|
||||
goto :eof
|
||||
|
||||
:put
|
||||
for %%F in ("%TARGET%\%~1") do if not exist "%%~dpF" mkdir "%%~dpF"
|
||||
copy /Y "%SOURCE%%~1" "%TARGET%\%~1" >nul
|
||||
if errorlevel 1 echo FAILED: %~1 & set /a COPYFAIL+=1
|
||||
goto :eof
|
||||
|
||||
:fail
|
||||
echo.
|
||||
echo Nothing was changed.
|
||||
pause
|
||||
exit /b 1
|
||||
Reference in new issue
Block a user