Add local Rebrickable catalogue mirror models
This commit is contained in:
jessikitty committed
1 parent
9d7fb66c6a
commit
63c3ece271
1 file changed
+182
@@ -0,0 +1,182 @@
|
||||
"""
|
||||
Local mirror of the Rebrickable parts catalogue.
|
||||
|
||||
This is reference data, not labels. The Update button fills it from
|
||||
Rebrickable's published CSV dumps so that creating a label is picking a part
|
||||
off a local list rather than hitting the API for every lookup.
|
||||
|
||||
Two tables:
|
||||
|
||||
* part_catalog_categories - Rebrickable's own part categories, kept
|
||||
separate from PartCategory (which is YOUR drawer taxonomy). The two are
|
||||
linked by an optional mapping so an import can drop parts straight into
|
||||
the right drawer group.
|
||||
* part_catalog_items - one row per Rebrickable part.
|
||||
|
||||
Roughly 60,000 parts land here. Everything is indexed for prefix search and
|
||||
nothing is fetched at render time.
|
||||
|
||||
Created by: Jess Rogerson (yelling commands at Claude.AI)
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from app import db
|
||||
|
||||
|
||||
class PartCatalogCategory(db.Model):
|
||||
"""A Rebrickable part category (their numbering, not yours)."""
|
||||
|
||||
__tablename__ = 'part_catalog_categories'
|
||||
|
||||
# Rebrickable's own id, used verbatim so re-syncs match on it.
|
||||
id = db.Column(db.Integer, primary_key=True, autoincrement=False)
|
||||
name = db.Column(db.String(200), nullable=False, index=True)
|
||||
|
||||
# Optional link into YOUR drawer taxonomy. Set it once and every part in
|
||||
# this Rebrickable category can be imported straight to the right place.
|
||||
drawer_category_id = db.Column(
|
||||
db.Integer, db.ForeignKey('part_categories.id'), nullable=True,
|
||||
index=True)
|
||||
|
||||
updated_at = db.Column(db.DateTime, nullable=False,
|
||||
default=datetime.utcnow, onupdate=datetime.utcnow)
|
||||
|
||||
drawer_category = db.relationship('PartCategory', foreign_keys=[drawer_category_id])
|
||||
items = db.relationship('PartCatalogItem', backref='catalog_category',
|
||||
lazy='dynamic')
|
||||
|
||||
def __repr__(self):
|
||||
return f'<PartCatalogCategory {self.id} {self.name}>'
|
||||
|
||||
def item_count(self):
|
||||
return self.items.count()
|
||||
|
||||
|
||||
class PartCatalogItem(db.Model):
|
||||
"""One part from Rebrickable's catalogue."""
|
||||
|
||||
__tablename__ = 'part_catalog_items'
|
||||
|
||||
id = db.Column(db.Integer, primary_key=True)
|
||||
|
||||
# Rebrickable's part_num. Unique, and the key a re-sync matches on.
|
||||
part_num = db.Column(db.String(32), unique=True, nullable=False,
|
||||
index=True)
|
||||
name = db.Column(db.String(300), nullable=False, index=True)
|
||||
category_id = db.Column(db.Integer,
|
||||
db.ForeignKey('part_catalog_categories.id'),
|
||||
nullable=True, index=True)
|
||||
part_material = db.Column(db.String(40), nullable=True)
|
||||
|
||||
# Derived once at sync time so the browse list can sort and filter on it
|
||||
# without re-parsing 60k names on every page load.
|
||||
dimensions = db.Column(db.String(40), nullable=True, index=True)
|
||||
|
||||
# Rebrickable CDN image. Not downloaded during a sync - that would be
|
||||
# 60,000 requests. Cached locally only when a part becomes a label.
|
||||
image_url = db.Column(db.String(500), nullable=True)
|
||||
|
||||
# Set when a PartLabel has been created from this row, so the browse
|
||||
# list can show what you have already labelled.
|
||||
has_label = db.Column(db.Boolean, nullable=False, default=False,
|
||||
index=True)
|
||||
|
||||
synced_at = db.Column(db.DateTime, nullable=False, default=datetime.utcnow)
|
||||
|
||||
def __repr__(self):
|
||||
return f'<PartCatalogItem {self.part_num} {self.name}>'
|
||||
|
||||
@property
|
||||
def display_name(self):
|
||||
return self.name
|
||||
|
||||
@property
|
||||
def rebrickable_url(self):
|
||||
return f'https://rebrickable.com/parts/{self.part_num}/'
|
||||
|
||||
@property
|
||||
def preview_url(self):
|
||||
"""
|
||||
A thumbnail for the browse list.
|
||||
|
||||
The CSV dumps carry no image column, and fetching 60,000 image URLs
|
||||
from the API during a sync would be absurd, so this falls back to
|
||||
Rebrickable's predictable LDraw render path. That 404s for parts
|
||||
with no LDraw model - the template hides broken thumbnails. Once a
|
||||
part becomes a label we fetch its real image_url via the API.
|
||||
"""
|
||||
if self.image_url:
|
||||
return self.image_url
|
||||
return ('https://cdn.rebrickable.com/media/parts/ldraw/15/'
|
||||
f'{self.part_num}.png')
|
||||
|
||||
def to_dict(self):
|
||||
return {
|
||||
'part_num': self.part_num,
|
||||
'name': self.name,
|
||||
'dimensions': self.dimensions,
|
||||
'image_url': self.image_url,
|
||||
'category': self.catalog_category.name if self.catalog_category else None,
|
||||
'has_label': self.has_label,
|
||||
}
|
||||
|
||||
|
||||
class CatalogSync(db.Model):
|
||||
"""
|
||||
One run of the Update button.
|
||||
|
||||
Kept so the page can show what happened last time, and so a sync that
|
||||
is still running can be polled for progress instead of blocking the
|
||||
request for several minutes.
|
||||
"""
|
||||
|
||||
__tablename__ = 'part_catalog_syncs'
|
||||
|
||||
id = db.Column(db.Integer, primary_key=True)
|
||||
started_at = db.Column(db.DateTime, nullable=False, default=datetime.utcnow)
|
||||
finished_at = db.Column(db.DateTime, nullable=True)
|
||||
|
||||
# running | done | failed
|
||||
status = db.Column(db.String(20), nullable=False, default='running')
|
||||
stage = db.Column(db.String(120), nullable=True)
|
||||
progress = db.Column(db.Integer, nullable=False, default=0) # 0-100
|
||||
|
||||
categories_added = db.Column(db.Integer, nullable=False, default=0)
|
||||
parts_added = db.Column(db.Integer, nullable=False, default=0)
|
||||
parts_updated = db.Column(db.Integer, nullable=False, default=0)
|
||||
message = db.Column(db.Text, nullable=True)
|
||||
|
||||
started_by_id = db.Column(db.Integer, db.ForeignKey('users.id'),
|
||||
nullable=True)
|
||||
started_by = db.relationship('User', foreign_keys=[started_by_id])
|
||||
|
||||
def __repr__(self):
|
||||
return f'<CatalogSync {self.id} {self.status}>'
|
||||
|
||||
@property
|
||||
def is_running(self):
|
||||
return self.status == 'running'
|
||||
|
||||
@property
|
||||
def duration(self):
|
||||
end = self.finished_at or datetime.utcnow()
|
||||
return (end - self.started_at).total_seconds()
|
||||
|
||||
def to_dict(self):
|
||||
return {
|
||||
'id': self.id,
|
||||
'status': self.status,
|
||||
'stage': self.stage,
|
||||
'progress': self.progress,
|
||||
'categories_added': self.categories_added,
|
||||
'parts_added': self.parts_added,
|
||||
'parts_updated': self.parts_updated,
|
||||
'message': self.message,
|
||||
'duration': round(self.duration),
|
||||
'finished': self.status != 'running',
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def latest():
|
||||
return CatalogSync.query.order_by(CatalogSync.id.desc()).first()
|
||||
Reference in new issue
Block a user