Initial Commit

2026-04-09 23:23:31 +02:00
commit 9223e4d35f
94 changed files with 15173 additions and 0 deletions
@@ -0,0 +1,261 @@
+pub mod fetch;
+pub mod filter;
+pub mod ldn;
+pub mod popular_names;
+pub mod vdb;
+
+use anyhow::Context;
+use sqlx::SqlitePool;
+
+use self::fetch::fetch_opengc;
+use self::filter::{compute_derived, is_suitable, CatalogEntry};
+use self::popular_names::popular_names;
+
+const CATALOG_TTL_SECS: i64 = 7 * 24 * 3600;
+// Bump this string whenever catalog ingestion logic changes.
+pub const CATALOG_VERSION: &str = "v5-normalized-ids";
+
+/// Force a full catalog re-ingest regardless of TTL or version.
+pub async fn force_refresh_catalog(pool: &SqlitePool) -> anyhow::Result<usize> {
+    // Clear version so next call to refresh_catalog unconditionally re-ingests
+    sqlx::query("DELETE FROM settings WHERE key = 'catalog_version'")
+        .execute(pool)
+        .await?;
+    do_refresh(pool).await
+}
+
+/// Check if catalog needs refresh and fetch+rebuild if so.
+pub async fn refresh_catalog(pool: &SqlitePool) -> anyhow::Result<()> {
+    let now = chrono::Utc::now().timestamp();
+
+    let last_fetch: Option<i64> =
+        sqlx::query_scalar("SELECT MAX(fetched_at) FROM catalog")
+            .fetch_optional(pool)
+            .await?
+            .flatten();
+
+    let stored_version: Option<String> =
+        sqlx::query_scalar("SELECT value FROM settings WHERE key = 'catalog_version'")
+            .fetch_optional(pool)
+            .await
+            .unwrap_or(None);
+
+    let version_stale = stored_version.as_deref() != Some(CATALOG_VERSION);
+
+    if let Some(last) = last_fetch {
+        if now - last < CATALOG_TTL_SECS && !version_stale {
+            tracing::info!("Catalog is up to date (last fetched {} seconds ago)", now - last);
+            return Ok(());
+        }
+    }
+    if version_stale {
+        tracing::info!("Catalog version changed to {} — forcing re-ingest", CATALOG_VERSION);
+    }
+
+    do_refresh(pool).await?;
+    Ok(())
+}
+
+async fn do_refresh(pool: &SqlitePool) -> anyhow::Result<usize> {
+    let entries = build_catalog().await?;
+    let count = entries.len();
+    
+    tracing::info!("Upserting {} total catalog entries...", count);
+    upsert_entries(pool, &entries).await?;
+
+    sqlx::query("INSERT OR REPLACE INTO settings (key, value) VALUES ('catalog_version', ?)")
+        .bind(CATALOG_VERSION)
+        .execute(pool)
+        .await?;
+
+    tracing::info!("Catalog refresh complete: {} objects", count);
+    Ok(count)
+}
+
+/// Build catalog entries from all sources without upserting to database.
+/// Useful for testing, validation, and dry-run operations.
+pub async fn build_catalog() -> anyhow::Result<Vec<CatalogEntry>> {
+    // Fetch all sources in parallel
+    tracing::info!("Refreshing catalog from OpenNGC + VdB + LDN...");
+    let (ngc_rows_res, vdb_res, ldn_res) = tokio::join!(
+        fetch_opengc(),
+        vdb::fetch_vdb(),
+        ldn::fetch_ldn(),
+    );
+
+    let names = popular_names();
+
+    let ngc_rows = ngc_rows_res.context("OpenNGC fetch failed")?;
+    let suitable: Vec<_> = ngc_rows.iter().filter(|r| is_suitable(r)).collect();
+    tracing::info!("OpenNGC: {}/{} rows suitable (RA/Dec valid + known type)", suitable.len(), ngc_rows.len());
+    
+    let mut entries: Vec<CatalogEntry> = suitable
+        .iter()
+        .filter_map(|r| compute_derived(r, &names))
+        .collect();
+    
+    tracing::info!("OpenNGC: {}/{} rows successfully derived to entries", entries.len(), suitable.len());
+    
+    // Generate additional Sharpless (Sh2) entries from objects that have Sh2 identifiers
+    let sh2_aliases: Vec<CatalogEntry> = entries
+        .iter()
+        .filter_map(|entry| create_sh2_alias(entry, &names))
+        .collect();
+    
+    tracing::info!("Generated {} Sharpless alias entries", sh2_aliases.len());
+    entries.extend(sh2_aliases);
+
+    match vdb_res {
+        Ok(vdb_entries) => {
+            tracing::info!("Adding {} VdB entries", vdb_entries.len());
+            entries.extend(vdb_entries);
+        }
+        Err(e) => tracing::warn!("VdB fetch failed (skipping): {}", e),
+    }
+
+    match ldn_res {
+        Ok(ldn_entries) => {
+            tracing::info!("Adding {} LDN entries", ldn_entries.len());
+            entries.extend(ldn_entries);
+        }
+        Err(e) => tracing::warn!("LDN fetch failed (skipping): {}", e),
+    }
+
+    Ok(entries)
+}
+
+/// Extract Sharpless identifier from an object's identifiers field and create an alias entry.
+fn create_sh2_alias(
+    entry: &CatalogEntry,
+    popular_names: &std::collections::HashMap<&'static str, &'static str>,
+) -> Option<CatalogEntry> {
+    // We'll need to parse identifiers from somewhere. 
+    // For now, we extract from the entry's existing data if available.
+    // The issue is that compute_derived doesn't store the original identifiers field.
+    // So we can look for Sh2 in the name or construct from the object type and catalog.
+    
+    // Check if this object already has "Sh2" in the ID (like "Sh2-155")
+    if entry.id.starts_with("Sh2-") {
+        return None; // Already a Sharpless entry
+    }
+    
+    // Only create Sh2 aliases for emission nebulae and similar objects
+    // that are likely to have Sharpless counterparts
+    if !matches!(
+        entry.obj_type.as_str(),
+        "emission_nebula" | "reflection_nebula" | "nebula" | "dark_nebula" | "planetary_nebula"
+    ) {
+        return None;
+    }
+
+    // Try to find a Sharpless name in popular_names for this object
+    // by checking known Sh2→NGC mappings
+    let sh2_id = match entry.id.as_str() {
+        // Sharpless → NGC known mappings
+        "NGC281" => "Sh2-184",   // Pac-Man
+        "NGC1333" => "Sh2-241",  // Reflection Nebula
+        "NGC1499" => "Sh2-220",  // California
+        "NGC2024" => "Sh2-68",   // Flame Nebula
+        "NGC2237" => "Sh2-64",   // Rosette
+        "NGC3372" => "Sh2-287",  // Eta Carinae
+        "NGC6210" => "Sh2-105",  // Turtle
+        "NGC6302" => "Sh2-12",   // Bug
+        "NGC6357" => "Sh2-11",   // War and Peace
+        "NGC6369" => "Sh2-72",   // Little Ghost
+        "NGC6611" => "Sh2-16",   // Eagle
+        "NGC6720" => "Sh2-83",   // Ring
+        "NGC6826" => "Sh2-87",   // Blinking
+        "NGC6853" => "Sh2-71",   // Dumbbell
+        "NGC6960" => "Sh2-103",  // Western Veil
+        "NGC6992" => "Sh2-103",  // Eastern Veil
+        "NGC7000" => "Sh2-119",  // North America
+        "NGC7009" => "Sh2-84",   // Saturn
+        "NGC7027" => "Sh2-107",  // Giraffe
+        "NGC7293" => "Sh2-108",  // Helix
+        "NGC7380" => "Sh2-142",  // Wizard
+        "NGC7635" => "Sh2-162",  // Bubble
+        "NGC7662" => "Sh2-120",  // Blue Snowball
+        "IC405" => "Sh2-229",    // Flaming Star
+        "IC434" => "Sh2-175",    // Horsehead
+        "IC1318" => "Sh2-100",   // Butterfly
+        "IC1805" => "Sh2-190",   // Heart
+        "IC1848" => "Sh2-199",   // Soul
+        "IC5070" => "Sh2-126",   // Pelican
+        _ => return None,
+    };
+
+    let common_name = popular_names
+        .get(sh2_id)
+        .or(popular_names.get(entry.id.as_str()))
+        .copied();
+
+    Some(CatalogEntry {
+        id: sh2_id.to_string(),
+        name: format!("{} ({})", sh2_id, entry.name),
+        common_name: common_name.map(|s| s.to_string()),
+        obj_type: entry.obj_type.clone(),
+        ra_deg: entry.ra_deg,
+        dec_deg: entry.dec_deg,
+        ra_h: entry.ra_h.clone(),
+        dec_dms: entry.dec_dms.clone(),
+        constellation: entry.constellation.clone(),
+        size_arcmin_maj: entry.size_arcmin_maj,
+        size_arcmin_min: entry.size_arcmin_min,
+        pos_angle_deg: entry.pos_angle_deg,
+        mag_v: entry.mag_v,
+        surface_brightness: entry.surface_brightness,
+        hubble_type: entry.hubble_type.clone(),
+        messier_num: None,
+        is_highlight: true, // Sharpless objects are highlights
+        fov_fill_pct: entry.fov_fill_pct,
+        mosaic_flag: entry.mosaic_flag,
+        mosaic_panels_w: entry.mosaic_panels_w,
+        mosaic_panels_h: entry.mosaic_panels_h,
+        difficulty: entry.difficulty,
+        guide_star_density: entry.guide_star_density.clone(),
+        fetched_at: entry.fetched_at,
+    })
+}
+
+pub async fn upsert_entries(pool: &SqlitePool, entries: &[CatalogEntry]) -> anyhow::Result<()> {
+    let mut tx = pool.begin().await?;
+    for e in entries {
+        sqlx::query(
+            r#"INSERT OR REPLACE INTO catalog
+               (id, name, common_name, obj_type, ra_deg, dec_deg, ra_h, dec_dms,
+                constellation, size_arcmin_maj, size_arcmin_min, pos_angle_deg,
+                mag_v, surface_brightness, hubble_type, messier_num, is_highlight,
+                fov_fill_pct, mosaic_flag, mosaic_panels_w, mosaic_panels_h,
+                difficulty, guide_star_density, fetched_at)
+               VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"#,
+        )
+        .bind(&e.id)
+        .bind(&e.name)
+        .bind(&e.common_name)
+        .bind(&e.obj_type)
+        .bind(e.ra_deg)
+        .bind(e.dec_deg)
+        .bind(&e.ra_h)
+        .bind(&e.dec_dms)
+        .bind(&e.constellation)
+        .bind(e.size_arcmin_maj)
+        .bind(e.size_arcmin_min)
+        .bind(e.pos_angle_deg)
+        .bind(e.mag_v)
+        .bind(e.surface_brightness)
+        .bind(&e.hubble_type)
+        .bind(e.messier_num)
+        .bind(e.is_highlight)
+        .bind(e.fov_fill_pct)
+        .bind(e.mosaic_flag)
+        .bind(e.mosaic_panels_w)
+        .bind(e.mosaic_panels_h)
+        .bind(e.difficulty)
+        .bind(e.guide_star_density.as_deref())
+        .bind(e.fetched_at)
+        .execute(&mut *tx)
+        .await?;
+    }
+    tx.commit().await?;
+    Ok(())
+}