update last
This commit is contained in:
adish-rmr
parent
2830d617df
commit
aec9ab8083
5 changed files with 128 additions and 29 deletions
66
CLAUDE.md
66
CLAUDE.md
|
|
@ -29,7 +29,7 @@ src/pif_compiler/
|
|||
│ ├── api_ingredients.py # Ingredient search by CAS + list all ingested + add tox indicator + clients CRUD
|
||||
│ ├── api_esposition.py # Esposition preset CRUD (create, list, delete)
|
||||
│ ├── api_orders.py # Order creation, retry, manual pipeline trigger, Excel/PDF export
|
||||
│ └── common.py # PDF generation, PubChem, CIR search endpoints
|
||||
│ └── common.py # PDF generation, PubChem, CIR search, segnalazione endpoints
|
||||
├── classes/
|
||||
│ ├── __init__.py # Re-exports all models from models.py and main_workflow.py
|
||||
│ ├── models.py # Pydantic models: Ingredient, DapInfo, CosingInfo,
|
||||
|
|
@ -39,6 +39,7 @@ src/pif_compiler/
|
|||
│ # orchestrator functions (receive_order, process_order_pipeline,
|
||||
│ # retry_order, trigger_pipeline)
|
||||
├── functions/
|
||||
│ ├── auth.py # JWT verification via Supabase JWKS (RS256/ES256), get_current_user FastAPI dependency
|
||||
│ ├── common_func.py # PDF generation with Playwright, tox+COSING source PDF batch generation, COSING PDF download, ZIP creation
|
||||
│ ├── common_log.py # Centralized logging configuration
|
||||
│ ├── db_utils.py # MongoDB + PostgreSQL connection helpers
|
||||
|
|
@ -68,7 +69,7 @@ src/pif_compiler/
|
|||
3. **ECHA** (`srv_echa.py`): Search substance -> get dossier -> parse HTML index -> extract toxicological data (NOAEL, LD50, LOAEL) from acute & repeated dose toxicity pages
|
||||
4. **PubChem** (`srv_pubchem.py`): Get molecular weight, XLogP, TPSA, melting point, dissociation constants
|
||||
5. **DAP calculation** (`DapInfo` model): Dermal Absorption Percentage based on molecular properties (MW > 500, LogP, TPSA > 120, etc.)
|
||||
6. **Toxicity ranking** (`Toxicity` model): Best toxicological indicator selection with priority (NOAEL > LOAEL > LD50) and safety factors
|
||||
6. **Toxicity ranking** (`Toxicity` model): Best toxicological indicator selection with 3-tier priority: (1) indicator type NOAEL=4 > LOAEL=3 > LD50=1; (2) route preference dermal > oral > inhalation > other; (3) lowest value for NOAEL/LOAEL (most conservative). Safety factors: LD50→10, LOAEL→3, NOAEL→1.
|
||||
|
||||
### Caching strategy
|
||||
- **ECHA results** are cached in MongoDB (`toxinfo.substance_index` collection) keyed by `substance.rmlCas`
|
||||
|
|
@ -108,7 +109,7 @@ POST /orders/create → receive_order() → BackgroundTasks → process_order_pi
|
|||
On error → stato=9 (ERRORE) + note with error message
|
||||
```
|
||||
|
||||
- **Order** (`main_workflow.py`): Pydantic model with DB table attributes + raw JSON from MongoDB. `pick_next()` classmethod picks the oldest pending order (FIFO). `validate_anagrafica()` upserts client in `clienti` table. `update_stato()` is the reusable state transition method.
|
||||
- **Order** (`main_workflow.py`): Pydantic model with DB table attributes + raw JSON from MongoDB. `pick_next()` classmethod picks the oldest pending order (FIFO). `validate_anagrafica()` upserts client in `clienti` table. `update_stato()` is the reusable state transition method. The compiler (`id_compilatore`) is resolved at order creation time via `receive_order(raw_json, compiler_name)` from the authenticated user's JWT — not during the pipeline.
|
||||
- **Project** (`main_workflow.py`): Created from Order via `Project.from_order()`. Holds `Esposition` preset (loaded by name from DB), list of `ProjectIngredient` with enriched `Ingredient` objects. `process_ingredients()` calls `Ingredient.get_or_create()` for each CAS. `save()` dumps to MongoDB `projects` collection, creates `progetti` entry, and populates `ingredients_lineage`.
|
||||
- **ProjectIngredient**: Helper model with cas, inci, percentage, is_colorante, skip_tox, and optional `Ingredient` object.
|
||||
- **Retry**: `retry_order(id_ordine)` resets an ERRORE order back to RICEVUTO for reprocessing.
|
||||
|
|
@ -130,9 +131,9 @@ Called via `Project.export_excel()` method, exposed at `GET /orders/export/{id_o
|
|||
|
||||
- `generate_project_source_pdfs(project)` — for each ingredient, generates two types of source PDFs:
|
||||
1. **Tox best_case PDF**: downloads the ECHA dossier page of `best_case` via Playwright. Naming: `CAS_source.pdf` (source is the `ToxIndicator.source` attribute, e.g., `56-81-5_repeated_dose_toxicity.pdf`)
|
||||
2. **COSING PDF**: downloads the official COSING regulation PDF via EU API for each `CosingInfo` with a `reference` attribute. Naming: `CAS_cosing.pdf`
|
||||
2. **COSING PDF**: one PDF per ingredient, using the first `CosingInfo` entry with a valid `reference`. Naming: `CAS_cosing.pdf`. Note: an ingredient may have multiple `CosingInfo` entries but only the first valid reference is used.
|
||||
- `cosing_download(ref_no)` — downloads the COSING regulation PDF from `api.tech.ec.europa.eu` by reference number. Returns PDF bytes or error string
|
||||
- `create_sources_zip(pdf_paths, zip_path)` — bundles all source PDFs into a ZIP archive
|
||||
- `create_sources_zip(pdf_paths, zip_path)` — bundles all source PDFs into a ZIP archive; deduplicates by filename to prevent duplicate entries
|
||||
- Exposed at `GET /orders/export-sources/{id_ordine}` — returns ZIP as FileResponse
|
||||
|
||||
### PostgreSQL schema (see `data/db_schema.sql`)
|
||||
|
|
@ -174,10 +175,15 @@ All routes are under `/api/v1`:
|
|||
| GET | `/orders/list` | List all orders with client/compiler/status info |
|
||||
| GET | `/orders/detail/{id_ordine}` | Full order detail with ingredients from MongoDB |
|
||||
| DELETE | `/orders/{id_ordine}` | Delete order and all related data (PostgreSQL + MongoDB) |
|
||||
| POST | `/auth/login` | Login with email+password, returns access+refresh token (public) |
|
||||
| POST | `/auth/refresh` | Refresh access token via refresh token (public) |
|
||||
| POST | `/auth/logout` | Invalidate session on Supabase (public) |
|
||||
| GET | `/auth/me` | Returns current user info from JWT (id, email, name) |
|
||||
| POST | `/common/pubchem` | PubChem property lookup by CAS |
|
||||
| POST | `/common/generate-pdf` | Generate PDF from URL via Playwright |
|
||||
| GET | `/common/download-pdf/{name}` | Download a generated PDF |
|
||||
| POST | `/common/cir-search` | CIR ingredient text search |
|
||||
| POST | `/common/segnalazione` | Save a user bug report/ticket to MongoDB `segnalazioni` collection |
|
||||
| GET | `/health`, `/ping` | Health check endpoints |
|
||||
|
||||
Docs available at `/docs` (Swagger) and `/redoc`.
|
||||
|
|
@ -191,6 +197,8 @@ Configured via `.env` file (loaded with `python-dotenv`):
|
|||
- `MONGO_HOST` - MongoDB host
|
||||
- `MONGO_PORT` - MongoDB port
|
||||
- `DATABASE_URL` - PostgreSQL connection string
|
||||
- `SUPABASE_URL` - Supabase project URL (e.g. `https://xxx.supabase.co`)
|
||||
- `SUPABASE_SECRET_KEY` - Supabase service role key (used for auth proxying)
|
||||
|
||||
## Development
|
||||
|
||||
|
|
@ -235,13 +243,16 @@ uv run uvicorn pif_compiler.main:app --reload --host 0.0.0.0 --port 8000
|
|||
- `upsert_cliente(nome_cliente)` - Upsert client, returns `id_cliente`
|
||||
- `upsert_compilatore(nome_compilatore)` - Upsert compiler, returns `id_compilatore`
|
||||
- `get_all_clienti()` - List all clients from PostgreSQL
|
||||
- `get_all_compilatori()` - List all compilers from PostgreSQL
|
||||
- `delete_cliente(nome_cliente)` - Delete client if no linked orders, returns None if blocked
|
||||
|
||||
**Orders:**
|
||||
- `insert_ordine(uuid_ordine, id_cliente)` - Insert new order, returns `id_ordine`
|
||||
- `insert_ordine(uuid_ordine, id_cliente=None, id_compilatore=None)` - Insert new order, returns `id_ordine`
|
||||
- `get_ordine_by_id(id_ordine)` - Get full order row
|
||||
- `get_oldest_pending_order()` - Get oldest order with stato=RICEVUTO
|
||||
- `aggiorna_stato_ordine(id_ordine, nuovo_stato)` - Update order status
|
||||
- `update_ordine_cliente(id_ordine, id_cliente)` - Set client on order
|
||||
- `update_ordine_compilatore(id_ordine, id_compilatore)` - Set compiler on order
|
||||
- `update_ordine_progetto(id_ordine, uuid_progetto)` - Set project UUID on order
|
||||
- `update_ordine_note(id_ordine, note)` - Set note on order
|
||||
- `reset_ordine_per_retry(id_ordine)` - Reset ERRORE order to RICEVUTO
|
||||
|
|
@ -257,12 +268,43 @@ uv run uvicorn pif_compiler.main:app --reload --host 0.0.0.0 --port 8000
|
|||
- `log_ricerche(cas, target, esito)` - Log search history
|
||||
|
||||
### Streamlit UI
|
||||
- `streamlit/ingredients_page.py` - Ingredient search by CAS + result display + inventory of ingested ingredients
|
||||
- `streamlit/exposition_page.py` - Esposition preset creation form + list of existing presets
|
||||
- `streamlit/order_page.py` - Order creation form (client dropdown, preset selection, ingredient data_editor with CAS/INCI/percentage, AQUA auto-detection, validation, submit with background processing)
|
||||
- `streamlit/orders_page.py` - Order management: list with filters (date, client, status), detail view with ingredients, actions (refresh, retry, Excel download, PDF sources ZIP, delete with confirmation), notes/log display
|
||||
- All pages call the FastAPI endpoints via `requests` (API must be running on `localhost:8000`)
|
||||
- Run with: `streamlit run streamlit/<page>.py`
|
||||
|
||||
Entry point: `streamlit/app.py` (multi-page app via `st.navigation`). Run with:
|
||||
```bash
|
||||
streamlit run streamlit/app.py
|
||||
```
|
||||
API must be running on `localhost:8000`.
|
||||
|
||||
#### Shared modules
|
||||
- `streamlit/functions.py` — single source of truth for all shared logic:
|
||||
- Constants: `API_BASE`, `AUTH_BASE`, `CAS_PATTERN`, `STATUS_MAP`, `WATER_INCI`
|
||||
- Auth: `do_login`, `do_refresh`, `do_logout`, `check_auth`, `_auth_headers`, `_fetch_user_info`
|
||||
- Cookie persistence: `get_cookie_manager()`, `_COOKIE_RT="pif_rt"`, `_COOKIE_MAX_AGE=7d` — uses `extra-streamlit-components` CookieManager. Only the `refresh_token` is stored in the browser cookie. `check_auth` restores session from cookie automatically on new browser sessions.
|
||||
- All API wrappers: `fetch_ingredient`, `fetch_orders`, `fetch_order_detail`, `download_excel`, `download_sources`, `send_segnalazione`, etc.
|
||||
- Order helpers: `validate_order`, `build_order_payload`, `make_empty_ingredient_df`, `is_water_inci`
|
||||
- ECHA extractors: `extract_tox_info_values`, `extract_acute_values`, `extract_repeated_values`
|
||||
- `streamlit/functions_ui.py` — UI-level helpers:
|
||||
- `search_cas_inci(input, type)` — DuckDB query on `streamlit/data.csv`
|
||||
- `search_cir(input_text)` — DuckDB query on `streamlit/cir-reports.csv`, returns `list[tuple[name, inci, url]]`
|
||||
- `show_login_page()` — login form calling `do_login`
|
||||
- `display_orderData(order_data)` — renders order detail
|
||||
|
||||
#### Pages (`streamlit/pages/`)
|
||||
- `ingredients_page.py` — ingredient search by CAS, displays DAP/COSING/tox data, PDF source download
|
||||
- `order_page.py` — order creation form: client, preset, ingredient table (INCI/CAS/%, AQUA auto-detection), submit → POST `/orders/create`
|
||||
- `list_orders.py` — order list with filters; detail view; retry/download/delete actions
|
||||
- `exposition_page.py` — exposure preset CRUD
|
||||
- `settings_page.py` — custom tox indicators, client management, ingredient inventory
|
||||
- `echa.py` — legacy ECHA direct search
|
||||
- `ticket.py` — bug report form → POST `/common/segnalazione`
|
||||
|
||||
#### Auth flow
|
||||
1. `get_cookie_manager()` is called at the very top of `app.py` before `check_auth` to render the JS component
|
||||
2. On login: tokens saved to `session_state` + `refresh_token` saved to cookie `pif_rt`
|
||||
3. On new session (tab reopen): `check_auth` reads cookie → calls `do_refresh` → restores session automatically
|
||||
4. On logout: `session_state` cleared + cookie deleted
|
||||
5. `_fetch_user_info()` called after login/restore → saves `user_name`, `user_email`, `user_id` to session_state
|
||||
6. Selected CAS/INCI saved to `session_state.selected_cas` / `session_state.selected_inci` for cross-page navigation
|
||||
|
||||
### Important domain concepts
|
||||
- **CAS number**: Chemical Abstracts Service identifier (e.g., "50-00-0")
|
||||
|
|
|
|||
|
|
@ -1,8 +1,9 @@
|
|||
import os
|
||||
import httpx
|
||||
from fastapi import APIRouter, HTTPException, status
|
||||
from fastapi import APIRouter, Depends, HTTPException, status
|
||||
from pydantic import BaseModel, EmailStr
|
||||
|
||||
from pif_compiler.functions.auth import get_current_user
|
||||
from pif_compiler.functions.common_log import get_logger
|
||||
|
||||
logger = get_logger()
|
||||
|
|
@ -89,6 +90,17 @@ async def refresh(request: RefreshRequest):
|
|||
}
|
||||
|
||||
|
||||
@router.get("/auth/me", tags=["Auth"])
|
||||
async def get_me(user: dict = Depends(get_current_user)):
|
||||
"""Ritorna le informazioni dell'utente autenticato dal JWT."""
|
||||
meta = user.get("user_metadata", {})
|
||||
return {
|
||||
"id": user.get("sub"),
|
||||
"email": user.get("email"),
|
||||
"name": meta.get("full_name") or meta.get("name"),
|
||||
}
|
||||
|
||||
|
||||
@router.post("/auth/logout", tags=["Auth"])
|
||||
async def logout(request: RefreshRequest):
|
||||
"""Invalida la sessione su Supabase tramite il refresh_token."""
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
from fastapi import APIRouter, HTTPException, BackgroundTasks, status
|
||||
from fastapi import APIRouter, Depends, HTTPException, BackgroundTasks, status
|
||||
from fastapi.responses import FileResponse
|
||||
from pydantic import BaseModel, Field
|
||||
from typing import List, Optional, Dict, Any
|
||||
|
|
@ -6,6 +6,7 @@ from typing import List, Optional, Dict, Any
|
|||
from pif_compiler.classes.main_workflow import receive_order, process_order_pipeline, retry_order, trigger_pipeline, Project
|
||||
from pif_compiler.functions.db_utils import db_connect, get_ordine_by_id, get_all_ordini, delete_ordine
|
||||
from pif_compiler.functions.common_func import generate_project_source_pdfs, create_sources_zip
|
||||
from pif_compiler.functions.auth import get_current_user
|
||||
from pif_compiler.functions.common_log import get_logger
|
||||
|
||||
logger = get_logger()
|
||||
|
|
@ -54,17 +55,24 @@ class OrderCreateResponse(BaseModel):
|
|||
# ==================== ROUTES ====================
|
||||
|
||||
@router.post("/orders/create", response_model=OrderCreateResponse, tags=["Orders"])
|
||||
async def create_order(request: OrderCreateRequest, background_tasks: BackgroundTasks):
|
||||
async def create_order(
|
||||
request: OrderCreateRequest,
|
||||
background_tasks: BackgroundTasks,
|
||||
user: dict = Depends(get_current_user),
|
||||
):
|
||||
"""
|
||||
Crea un nuovo ordine e avvia l'elaborazione in background.
|
||||
Il JSON viene salvato su MongoDB, il record su PostgreSQL (stato=RICEVUTO).
|
||||
L'arricchimento degli ingredienti avviene in background.
|
||||
Il compilatore viene valorizzato automaticamente dall'utente autenticato.
|
||||
"""
|
||||
logger.info(f"Nuovo ordine ricevuto: cliente={request.client_name}, prodotto={request.product_name}")
|
||||
meta = user.get("user_metadata", {})
|
||||
compiler_name = meta.get("full_name") or meta.get("name") or user.get("email") or user.get("sub")
|
||||
logger.info(f"Nuovo ordine ricevuto: cliente={request.client_name}, prodotto={request.product_name}, compilatore={compiler_name}")
|
||||
|
||||
try:
|
||||
raw_json = request.model_dump()
|
||||
id_ordine = receive_order(raw_json)
|
||||
id_ordine = receive_order(raw_json, compiler_name=compiler_name)
|
||||
|
||||
if id_ordine is None:
|
||||
return OrderCreateResponse(
|
||||
|
|
|
|||
|
|
@ -14,9 +14,9 @@ from pif_compiler.classes.models import (
|
|||
StatoOrdine, Ingredient, Esposition
|
||||
)
|
||||
from pif_compiler.functions.db_utils import (
|
||||
db_connect, upsert_cliente, aggiorna_stato_ordine,
|
||||
db_connect, upsert_cliente, upsert_compilatore, aggiorna_stato_ordine,
|
||||
insert_ordine, get_oldest_pending_order,
|
||||
update_ordine_cliente, update_ordine_progetto, update_ordine_note,
|
||||
update_ordine_cliente, update_ordine_compilatore, update_ordine_progetto, update_ordine_note,
|
||||
get_preset_id_by_name, insert_progetto,
|
||||
insert_ingredient_lineage, get_ingrediente_id_by_cas,
|
||||
get_ordine_by_id, reset_ordine_per_retry
|
||||
|
|
@ -264,25 +264,33 @@ class Project(BaseModel):
|
|||
|
||||
# ==================== ORCHESTRATOR ====================
|
||||
|
||||
def receive_order(raw_json: dict) -> Optional[int]:
|
||||
def receive_order(raw_json: dict, compiler_name: Optional[str] = None) -> Optional[int]:
|
||||
"""
|
||||
Riceve un ordine dal front-end, lo salva su MongoDB e crea il record in PostgreSQL.
|
||||
Se compiler_name è fornito, crea/recupera il compilatore e lo lega subito all'ordine.
|
||||
Ritorna id_ordine.
|
||||
"""
|
||||
# 1. Salva il JSON grezzo su MongoDB collection 'orders'
|
||||
# 1. Risolvi il compilatore prima di salvare, così è già nel record
|
||||
id_compilatore = None
|
||||
if compiler_name:
|
||||
id_compilatore = upsert_compilatore(compiler_name)
|
||||
if id_compilatore is None:
|
||||
logger.warning(f"receive_order: upsert compilatore '{compiler_name}' fallito, ordine creato senza compilatore")
|
||||
|
||||
# 2. Salva il JSON grezzo su MongoDB collection 'orders'
|
||||
collection = db_connect(collection_name='orders')
|
||||
result = collection.insert_one(raw_json.copy()) # copy per evitare side-effects su _id
|
||||
result = collection.insert_one(raw_json.copy())
|
||||
uuid_ordine = str(result.inserted_id)
|
||||
|
||||
logger.info(f"Ordine salvato su MongoDB: uuid_ordine={uuid_ordine}")
|
||||
|
||||
# 2. Crea il record nella tabella ordini (stato = RICEVUTO)
|
||||
id_ordine = insert_ordine(uuid_ordine)
|
||||
# 3. Crea il record nella tabella ordini (stato = RICEVUTO) con compilatore già valorizzato
|
||||
id_ordine = insert_ordine(uuid_ordine, id_compilatore=id_compilatore)
|
||||
if id_ordine is None:
|
||||
logger.error(f"Errore creazione record ordini per uuid={uuid_ordine}")
|
||||
return None
|
||||
|
||||
logger.info(f"Ordine {id_ordine} creato in PostgreSQL (stato=RICEVUTO)")
|
||||
logger.info(f"Ordine {id_ordine} creato in PostgreSQL (stato=RICEVUTO, compilatore={'id=' + str(id_compilatore) if id_compilatore else 'n/d'})")
|
||||
return id_ordine
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -170,6 +170,20 @@ def get_all_clienti():
|
|||
logger.error(f"Errore recupero clienti: {e}")
|
||||
return []
|
||||
|
||||
def get_all_compilatori():
|
||||
"""Recupera tutti i compilatori dalla tabella compilatori."""
|
||||
try:
|
||||
conn = postgres_connect()
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("SELECT id_compilatore, nome_compilatore FROM compilatori ORDER BY nome_compilatore")
|
||||
results = cur.fetchall()
|
||||
conn.close()
|
||||
return results if results else []
|
||||
except Exception as e:
|
||||
logger.error(f"Errore recupero compilatori: {e}")
|
||||
return []
|
||||
|
||||
|
||||
def delete_cliente(nome_cliente: str) -> bool:
|
||||
"""Elimina un cliente per nome. Ritorna None se ha ordini collegati."""
|
||||
try:
|
||||
|
|
@ -203,7 +217,7 @@ def log_ricerche(cas, target, esito):
|
|||
logger.error(f"Error: {e}")
|
||||
return
|
||||
|
||||
def insert_ordine(uuid_ordine, id_cliente=None):
|
||||
def insert_ordine(uuid_ordine, id_cliente=None, id_compilatore=None):
|
||||
"""Inserisce un nuovo ordine nella tabella ordini. Ritorna id_ordine."""
|
||||
from datetime import datetime as dt
|
||||
from pif_compiler.classes.models import StatoOrdine
|
||||
|
|
@ -211,9 +225,9 @@ def insert_ordine(uuid_ordine, id_cliente=None):
|
|||
conn = postgres_connect()
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""INSERT INTO ordini (uuid_ordine, id_cliente, data_ordine, stato_ordine)
|
||||
VALUES (%s, %s, %s, %s) RETURNING id_ordine;""",
|
||||
(uuid_ordine, id_cliente, dt.now(), int(StatoOrdine.RICEVUTO))
|
||||
"""INSERT INTO ordini (uuid_ordine, id_cliente, id_compilatore, data_ordine, stato_ordine)
|
||||
VALUES (%s, %s, %s, %s, %s) RETURNING id_ordine;""",
|
||||
(uuid_ordine, id_cliente, id_compilatore, dt.now(), int(StatoOrdine.RICEVUTO))
|
||||
)
|
||||
result = cur.fetchone()
|
||||
conn.commit()
|
||||
|
|
@ -245,6 +259,21 @@ def get_oldest_pending_order():
|
|||
logger.error(f"Errore recupero ordine pendente: {e}")
|
||||
return None
|
||||
|
||||
def update_ordine_compilatore(id_ordine, id_compilatore):
|
||||
"""Aggiorna id_compilatore sull'ordine."""
|
||||
try:
|
||||
conn = postgres_connect()
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"UPDATE ordini SET id_compilatore = %s WHERE id_ordine = %s",
|
||||
(id_compilatore, id_ordine)
|
||||
)
|
||||
conn.commit()
|
||||
conn.close()
|
||||
except Exception as e:
|
||||
logger.error(f"Errore aggiornamento compilatore ordine {id_ordine}: {e}")
|
||||
|
||||
|
||||
def update_ordine_cliente(id_ordine, id_cliente):
|
||||
"""Aggiorna id_cliente sull'ordine."""
|
||||
try:
|
||||
|
|
|
|||
Loading…
Reference in a new issue