from datetime import date, datetime, time, timedelta
import json
from pathlib import Path
import ssl
import tempfile
import xml.etree.ElementTree as ET
from typing import List, Optional, Tuple
import gradio as gr
import folium
from folium.plugins import MarkerCluster
import pandas as pd
from huggingface_hub import hf_hub_download
try:
from gradio.components import Date as GrDateComponent
except (ImportError, AttributeError):
GrDateComponent = getattr(gr, "Date", None) or getattr(gr, "DatePicker", None)
try:
from shapely import wkt as shapely_wkt
from shapely.geometry import Point
SHAPELY_AVAILABLE = True
except Exception: # ImportError or attribute issues
shapely_wkt = None
Point = None
SHAPELY_AVAILABLE = False
DEFAULT_CENTER = "41.9028,12.4964"
DEFAULT_ZOOM = 12
DEFAULT_TILES = "CartoDB positron"
DEFAULT_DATE_PROMPT = "Select the date to pull AIS data."
DEFAULT_TIME_PROMPT = "Set start and end times to describe the daily window."
DEFAULT_DATE = "2025-08-25"
DEFAULT_START_TIME = "10:00:00"
DEFAULT_END_TIME = "12:00:00"
DEFAULT_AOI_WKT = """POLYGON((4.2100 51.3700,4.4800 51.3700,4.5100 51.2900,4.4650 51.1700,4.2500 51.1700,4.1900 51.2500,4.2100 51.3700))"""
HF_REPO_ID = "Lore0123/AISPortal"
HF_FILE_TEMPLATE = "{date}_ais.parquet"
DATE_FMT = "%Y-%m-%d"
DEFAULT_DATE_OBJ = datetime.strptime(DEFAULT_DATE, DATE_FMT).date()
MAX_POINTS = 10_000
BANNER_PATH = (Path(__file__).resolve().parent / "src" / "banner.png")
TILE_OPTIONS = {
"OpenStreetMap": {
"tiles": "OpenStreetMap",
"attr": "© OpenStreetMap contributors",
},
"Stamen Terrain": {
"tiles": "Stamen Terrain",
"attr": "Map tiles by Stamen Design, CC BY 3.0 — Data © OpenStreetMap contributors",
},
"CartoDB positron": {
"tiles": "https://{s}.basemaps.cartocdn.com/light_all/{z}/{x}/{y}{r}.png",
"attr": "© OpenStreetMap contributors © CARTO",
},
"CartoDB dark_matter": {
"tiles": "https://{s}.basemaps.cartocdn.com/dark_all/{z}/{x}/{y}{r}.png",
"attr": "© OpenStreetMap contributors © CARTO",
},
}
def _parse_center(center: str) -> Tuple[float, float]:
"""
Parse "lat,lon" into (lat, lon).
"""
try:
lat_str, lon_str = [x.strip() for x in center.split(",")]
lat, lon = float(lat_str), float(lon_str)
if not (-90 <= lat <= 90 and -180 <= lon <= 180):
raise ValueError
return lat, lon
except Exception:
# Default: Rome
return 41.9028, 12.4964
def _parse_date(value) -> Optional[date]:
if not value:
return None
if isinstance(value, date):
return value
if isinstance(value, str):
raw = value.strip()
if not raw:
return None
try:
return datetime.strptime(raw, DATE_FMT).date()
except ValueError:
return None
return None
def _iterate_dates(start: Optional[date], end: Optional[date]) -> List[date]:
if start and end:
if end < start:
start, end = end, start
elif start:
end = start
elif end:
start = end
else:
return []
current = start
dates: List[date] = []
while current <= end:
dates.append(current)
current += timedelta(days=1)
return dates
def _normalize_column_key(value: str) -> str:
return "".join(ch for ch in value.lower() if ch.isalnum())
def _find_column(df: pd.DataFrame, candidates: List[str]) -> Optional[str]:
normalized_map = {}
for col in df.columns:
normalized_map.setdefault(_normalize_column_key(col), col)
for candidate in candidates:
key = _normalize_column_key(candidate)
if key in normalized_map:
return normalized_map[key]
return None
def _parse_time(value: Optional[str]) -> Optional[time]:
if not value:
return None
if isinstance(value, str):
raw = value.strip()
if not raw:
return None
for fmt in ("%H:%M:%S", "%H:%M"):
try:
parsed = datetime.strptime(raw, fmt)
return parsed.time()
except ValueError:
continue
return None
return None
def _build_time_mask(datetimes: pd.Series,
start_time_obj: Optional[time],
end_time_obj: Optional[time]) -> Optional[pd.Series]:
if start_time_obj is None and end_time_obj is None:
return None
dt_series = pd.to_datetime(datetimes, errors="coerce", utc=False)
valid = dt_series.notna()
times = dt_series.dt.time
cond = pd.Series(True, index=dt_series.index)
if start_time_obj and end_time_obj:
if start_time_obj <= end_time_obj:
cond &= (times >= start_time_obj) & (times <= end_time_obj)
else:
cond &= (times >= start_time_obj) | (times <= end_time_obj)
elif start_time_obj:
cond &= times >= start_time_obj
else:
cond &= times <= end_time_obj
return cond & valid
def _load_ais_points(start_date: Optional[str],
end_date: Optional[str],
start_time: Optional[str],
end_time: Optional[str]) -> Tuple[pd.DataFrame, List[str]]:
"""Download AIS parquet files, filter them, and return the full filtered rows."""
start = _parse_date(start_date)
end = _parse_date(end_date)
dates = _iterate_dates(start, end)
if not dates:
return pd.DataFrame(columns=["name", "lat", "lon", "source_date", "timestamp", "mmsi"]), []
frames: List[pd.DataFrame] = []
errors: List[str] = []
start_time_obj = _parse_time(start_time)
end_time_obj = _parse_time(end_time)
for day in dates:
filename = HF_FILE_TEMPLATE.format(date=day.isoformat())
try:
local_path = hf_hub_download(
repo_id=HF_REPO_ID,
filename=filename,
repo_type="dataset"
)
except Exception as exc: # pragma: no cover - network dependent
errors.append(f"{day}: download failed ({exc})")
continue
try:
df = pd.read_parquet(local_path)
except Exception as exc: # pragma: no cover - file dependent
errors.append(f"{day}: failed to read parquet ({exc})")
continue
lat_col = _find_column(df, ["lat", "latitude"])
lon_col = _find_column(df, ["lon", "longitude", "long", "lng"])
if lat_col is None or lon_col is None:
errors.append(f"{day}: missing latitude/longitude columns")
continue
time_col = _find_column(df, [
"tstamp",
"timestamp",
"time",
"datetime",
"basedatetime",
"baseDateTime",
"received_time",
"receivedtime"
])
if time_col is not None:
mask = _build_time_mask(df[time_col], start_time_obj, end_time_obj)
if mask is not None:
df = df[mask.fillna(False)]
elif start_time_obj or end_time_obj:
errors.append(f"{day}: no timestamp column for time filtering")
if df.empty:
continue
lat_series = pd.to_numeric(df[lat_col], errors="coerce")
lon_series = pd.to_numeric(df[lon_col], errors="coerce")
valid_mask = lat_series.notna() & lon_series.notna()
if not valid_mask.any():
continue
subset = df.loc[valid_mask].copy()
subset["lat"] = lat_series.loc[valid_mask].astype(float)
subset["lon"] = lon_series.loc[valid_mask].astype(float)
name_col = _find_column(df, ["name", "shipname", "vessel", "imo", "callsign", "vesselname"])
if name_col is not None:
subset_names = subset[name_col].fillna("").astype(str)
else:
subset_names = pd.Series("", index=subset.index)
subset["name"] = subset_names.replace({"nan": "", "None": ""})
subset["source_date"] = day.isoformat()
mmsi_col = _find_column(df, ["mmsi", "mmsi_id"])
if mmsi_col is not None:
subset_mmsi = subset[mmsi_col].fillna("").astype(str)
subset_mmsi = subset_mmsi.replace({"nan": "", "None": ""})
subset["mmsi"] = subset_mmsi
else:
subset["mmsi"] = ""
if time_col is not None:
ts_series = pd.to_datetime(subset[time_col], errors="coerce", utc=True)
try:
ts_local = ts_series.dt.tz_convert(None)
except TypeError: # already naive
ts_local = ts_series
subset["timestamp"] = ts_local.dt.strftime("%Y-%m-%d %H:%M:%S").fillna("")
else:
subset["timestamp"] = ""
frames.append(subset.reset_index(drop=True))
if not frames:
return pd.DataFrame(columns=[
"name",
"lat",
"lon",
"source_date",
"timestamp",
"mmsi"
]), errors
result = pd.concat(frames, ignore_index=True)
return result, errors
def render_map(selected_date,
start_time: Optional[str],
end_time: Optional[str],
aoi_wkt: Optional[str]) -> Tuple[str, str, str]:
"""
Build a Leaflet map and return full HTML (rendered by Gradio HTML component).
"""
lat, lon = _parse_center(DEFAULT_CENTER)
tile_cfg = TILE_OPTIONS[DEFAULT_TILES]
map_kwargs = {
"location": [lat, lon],
"zoom_start": DEFAULT_ZOOM,
"tiles": tile_cfg.get("tiles", DEFAULT_TILES),
"control_scale": True,
"width": "100%",
"height": "600px",
}
attr = tile_cfg.get("attr")
if attr:
map_kwargs["attr"] = attr
m = folium.Map(**map_kwargs)
# Points
bounds: List[Tuple[float, float]] = []
point_count = 0
error_message: Optional[str] = None
error_marker_added = False
selected_date_str = _coerce_date_string(selected_date)
export_df = pd.DataFrame()
try:
export_df, errors = _load_ais_points(selected_date_str, selected_date_str, start_time, end_time)
if not export_df.empty:
export_df, aoi_error = _filter_by_aoi(export_df, aoi_wkt)
if aoi_error:
errors.append(aoi_error)
map_df = pd.DataFrame()
if not export_df.empty:
map_df = export_df[["name", "lat", "lon", "source_date", "timestamp", "mmsi"]].copy()
if len(map_df) > MAX_POINTS:
sampled_idx = map_df.sample(MAX_POINTS, random_state=0).index
map_df = map_df.loc[sampled_idx]
map_df = map_df.reset_index(drop=True)
if not map_df.empty:
cluster = MarkerCluster(name="AIS Points").add_to(m)
for _, r in map_df.iterrows():
name_raw = r.get("name")
name = str(name_raw).strip() if name_raw is not None else ""
if name.lower() == "nan":
name = ""
source_date = r.get("source_date", "?")
timestamp = r.get("timestamp")
mmsi = str(r.get("mmsi") or "").strip()
details = []
if name:
details.append(f"Name: {name}")
if mmsi:
details.append(f"MMSI: {mmsi}")
details.append(f"Date: {source_date}")
if isinstance(timestamp, str) and timestamp:
details.append(f"Timestamp: {timestamp}")
details.append(f"Lat: {r['lat']:.6f}")
details.append(f"Lon: {r['lon']:.6f}")
popup = "
".join(details)
folium.Marker([r["lat"], r["lon"]], popup=popup).add_to(cluster)
bounds.append((r["lat"], r["lon"]))
point_count = len(map_df)
error_message = _summarize_errors(errors)
except Exception as e:
error_message = f"AIS data error: {e}"
_add_error_marker(m, lat, lon, error_message)
error_marker_added = True
if error_message and not error_marker_added:
_add_error_marker(m, lat, lon, error_message)
# Fit to data if any bounds collected
if bounds:
m.fit_bounds(bounds, padding=(20, 20))
html = m._repr_html_()
date_range = _format_date_display(selected_date_str, default_prompt=DEFAULT_DATE_PROMPT)
time_range = _format_range(start_time, end_time, default_prompt=DEFAULT_TIME_PROMPT)
info_lines = [
"### Selected Period",
f"- Date: {date_range}",
f"- Times: {time_range}",
f"- Points on map: {point_count}"
]
if error_message:
info_lines.append(f"- Error: {error_message}")
ssl_msg = _ssl_warning()
if ssl_msg:
info_lines.append(f"- SSL: {ssl_msg}")
export_payload = export_df.reset_index(drop=True)
data_json = export_payload.to_json(orient="records") if not export_payload.empty else "[]"
return html, "\n".join(info_lines), data_json
def _format_range(start: Optional[str], end: Optional[str], default_prompt: str) -> str:
start_clean = _clean_input(start)
end_clean = _clean_input(end)
if not start_clean and not end_clean:
return default_prompt
return f"{start_clean or '—'} → {end_clean or '—'}"
def _clean_input(value: Optional[str]) -> Optional[str]:
if value is None:
return None
if isinstance(value, str):
cleaned = value.strip()
return cleaned or None
return str(value)
def _filter_by_aoi(df: pd.DataFrame, wkt_text: Optional[str]) -> Tuple[pd.DataFrame, Optional[str]]:
wkt_clean = _clean_input(wkt_text)
if not wkt_clean:
return df, None
if not SHAPELY_AVAILABLE or shapely_wkt is None or Point is None:
return df, "AOI filter unavailable: install shapely."
try:
geom = shapely_wkt.loads(wkt_clean)
except Exception as exc:
return df, f"AOI parse error: {exc}"
if geom.is_empty:
return df, "AOI geometry is empty."
def contains_point(row) -> bool:
try:
pt = Point(float(row["lon"]), float(row["lat"]))
except Exception:
return False
return geom.contains(pt) or geom.touches(pt)
mask = df.apply(contains_point, axis=1)
if mask.sum() == 0:
return df.iloc[0:0].copy(), "AOI filter removed all points."
return df[mask].reset_index(drop=True), None
def _summarize_errors(errors: List[str]) -> Optional[str]:
if not errors:
return None
unique: List[str] = []
for err in errors:
if err not in unique:
unique.append(err)
if len(unique) == 3:
break
extra = len(errors) - len(unique)
message = "; ".join(unique)
if extra > 0:
message += f"; (+{extra} more)"
return message
def _add_error_marker(map_obj: folium.Map, lat: float, lon: float, message: str) -> None:
folium.Marker(
[lat, lon],
icon=folium.DivIcon(html=f"