Spaces:
Sleeping
Sleeping
Hardik Singh commited on
Commit ·
bc1ac55
1
Parent(s): 124cd04
Frontier Overhaul
Browse files- api/bootstrap.py +5 -1
- api/routes/conflicts.py +21 -43
- frontend/src/Dashboard.jsx +0 -4
- frontend/src/components/IncidentIntelligenceCard.jsx +92 -103
- frontend/src/components/TacticalFeed.jsx +10 -0
- frontend/src/components/TacticalMap.jsx +140 -36
- frontend/src/index.css +30 -0
- poller/classifier.py +45 -26
- poller/db_inserter.py +43 -4
- poller/deduplicator.py +29 -0
- poller/gdelt_poller.py +41 -12
- poller/geo_utils.py +56 -6
- poller/rss_poller.py +69 -44
api/bootstrap.py
CHANGED
|
@@ -112,10 +112,14 @@ async def bootstrap_db():
|
|
| 112 |
ADD COLUMN IF NOT EXISTS location_raw TEXT,
|
| 113 |
ADD COLUMN IF NOT EXISTS location_admin1 VARCHAR(100),
|
| 114 |
ADD COLUMN IF NOT EXISTS geo_validation_flags TEXT[],
|
| 115 |
-
ADD COLUMN IF NOT EXISTS ai_analysis TEXT
|
|
|
|
|
|
|
|
|
|
| 116 |
|
| 117 |
CREATE INDEX IF NOT EXISTS idx_geo_confidence ON conflict_events(geo_confidence);
|
| 118 |
CREATE INDEX IF NOT EXISTS idx_ai_analysis ON conflict_events USING GIN(to_tsvector('english', COALESCE(ai_analysis, '')));
|
|
|
|
| 119 |
""")
|
| 120 |
|
| 121 |
log.info("[Bootstrap] Database schema validation and initialization complete.")
|
|
|
|
| 112 |
ADD COLUMN IF NOT EXISTS location_raw TEXT,
|
| 113 |
ADD COLUMN IF NOT EXISTS location_admin1 VARCHAR(100),
|
| 114 |
ADD COLUMN IF NOT EXISTS geo_validation_flags TEXT[],
|
| 115 |
+
ADD COLUMN IF NOT EXISTS ai_analysis TEXT,
|
| 116 |
+
ADD COLUMN IF NOT EXISTS verification_count INTEGER DEFAULT 1,
|
| 117 |
+
ADD COLUMN IF NOT EXISTS source_urls TEXT[] DEFAULT '{}',
|
| 118 |
+
ADD COLUMN IF NOT EXISTS strategic_relevance VARCHAR(15) DEFAULT 'LOW';
|
| 119 |
|
| 120 |
CREATE INDEX IF NOT EXISTS idx_geo_confidence ON conflict_events(geo_confidence);
|
| 121 |
CREATE INDEX IF NOT EXISTS idx_ai_analysis ON conflict_events USING GIN(to_tsvector('english', COALESCE(ai_analysis, '')));
|
| 122 |
+
CREATE INDEX IF NOT EXISTS idx_verification ON conflict_events(verification_count DESC);
|
| 123 |
""")
|
| 124 |
|
| 125 |
log.info("[Bootstrap] Database schema validation and initialization complete.")
|
api/routes/conflicts.py
CHANGED
|
@@ -7,6 +7,20 @@ from fastapi.encoders import jsonable_encoder
|
|
| 7 |
|
| 8 |
router = APIRouter(prefix="/conflicts", tags=["Conflicts"])
|
| 9 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 10 |
async def check_cache(request: Request, cache_key: str):
|
| 11 |
if db.redis:
|
| 12 |
cached = await db.redis.get(cache_key)
|
|
@@ -76,15 +90,7 @@ async def get_conflicts(
|
|
| 76 |
|
| 77 |
data = [dict(r) for r in records]
|
| 78 |
for d in data:
|
| 79 |
-
|
| 80 |
-
d["event_time"] = d["event_time"].isoformat() + "Z" if d["event_time"] else None
|
| 81 |
-
d["event_date"] = d["event_date"].isoformat() if d["event_date"] else None
|
| 82 |
-
d["ingested_at"] = d["ingested_at"].isoformat() + "Z" if d.get("ingested_at") else None
|
| 83 |
-
# New Precision Metadata
|
| 84 |
-
d["geo_confidence"] = float(d.get("geo_confidence") or 0.0)
|
| 85 |
-
d["is_approximate"] = d.get("geo_precision", 3) >= 3
|
| 86 |
-
d["ai_analysis"] = d.get("ai_analysis")
|
| 87 |
-
if d.get("geom"): del d["geom"]
|
| 88 |
|
| 89 |
response = {
|
| 90 |
"status": 200,
|
|
@@ -117,13 +123,7 @@ async def get_recent_conflicts(request: Request, days: int = 7, limit: int = 100
|
|
| 117 |
|
| 118 |
data = [dict(r) for r in records]
|
| 119 |
for d in data:
|
| 120 |
-
|
| 121 |
-
d["event_date"] = d["event_date"].isoformat() if d["event_date"] else None
|
| 122 |
-
d["ingested_at"] = d["ingested_at"].isoformat() + "Z" if d.get("ingested_at") else None
|
| 123 |
-
d["geo_confidence"] = float(d.get("geo_confidence") or 0.0)
|
| 124 |
-
d["is_approximate"] = d.get("geo_precision", 3) >= 3
|
| 125 |
-
d["ai_analysis"] = d.get("ai_analysis")
|
| 126 |
-
if d.get("geom"): del d["geom"]
|
| 127 |
|
| 128 |
response = {
|
| 129 |
"status": 200,
|
|
@@ -152,10 +152,7 @@ async def get_ongoing_conflicts(request: Request, limit: int = 50):
|
|
| 152 |
|
| 153 |
data = [dict(r) for r in records]
|
| 154 |
for d in data:
|
| 155 |
-
|
| 156 |
-
d["geo_confidence"] = float(d.get("geo_confidence") or 0.0)
|
| 157 |
-
d["is_approximate"] = d.get("geo_precision", 3) >= 3
|
| 158 |
-
if d.get("geom"): del d["geom"]
|
| 159 |
|
| 160 |
res = {"status": 200, "success": True, "count": len(data), "data": data}
|
| 161 |
await set_cache(cache_key, res, ttl=60) # Short TTL for live data
|
|
@@ -181,10 +178,7 @@ async def get_historical_conflicts(request: Request, days_ago: int = 2, limit: i
|
|
| 181 |
|
| 182 |
data = [dict(r) for r in records]
|
| 183 |
for d in data:
|
| 184 |
-
|
| 185 |
-
d["geo_confidence"] = float(d.get("geo_confidence") or 0.0)
|
| 186 |
-
d["is_approximate"] = d.get("geo_precision", 3) >= 3
|
| 187 |
-
if d.get("geom"): del d["geom"]
|
| 188 |
|
| 189 |
res = {"status": 200, "success": True, "count": len(data), "data": data}
|
| 190 |
await set_cache(cache_key, res, ttl=3600) # Long TTL for history
|
|
@@ -218,12 +212,7 @@ async def get_conflicts_near(
|
|
| 218 |
|
| 219 |
data = [dict(r) for r in records]
|
| 220 |
for d in data:
|
| 221 |
-
|
| 222 |
-
d["event_date"] = d["event_date"].isoformat() if d["event_date"] else None
|
| 223 |
-
d["ingested_at"] = d["ingested_at"].isoformat() + "Z" if d.get("ingested_at") else None
|
| 224 |
-
d["geo_confidence"] = float(d.get("geo_confidence") or 0.0)
|
| 225 |
-
d["is_approximate"] = d.get("geo_precision", 3) >= 3
|
| 226 |
-
if d.get("geom"): del d["geom"]
|
| 227 |
|
| 228 |
response = {
|
| 229 |
"status": 200,
|
|
@@ -251,12 +240,7 @@ async def get_conflicts_country(request: Request, iso3: str, days: int = 30, lim
|
|
| 251 |
|
| 252 |
data = [dict(r) for r in records]
|
| 253 |
for d in data:
|
| 254 |
-
|
| 255 |
-
d["event_date"] = d["event_date"].isoformat() if d["event_date"] else None
|
| 256 |
-
d["ingested_at"] = d["ingested_at"].isoformat() + "Z" if d.get("ingested_at") else None
|
| 257 |
-
d["geo_confidence"] = float(d.get("geo_confidence") or 0.0)
|
| 258 |
-
d["is_approximate"] = d.get("geo_precision", 3) >= 3
|
| 259 |
-
if d.get("geom"): del d["geom"]
|
| 260 |
|
| 261 |
response = {
|
| 262 |
"status": 200,
|
|
@@ -278,13 +262,7 @@ async def get_conflict_detail(event_id: str):
|
|
| 278 |
raise HTTPException(status_code=404, detail="Event not found")
|
| 279 |
|
| 280 |
d = dict(record)
|
| 281 |
-
|
| 282 |
-
d["event_date"] = d["event_date"].isoformat() if d["event_date"] else None
|
| 283 |
-
d["ingested_at"] = d["ingested_at"].isoformat() + "Z" if d.get("ingested_at") else None
|
| 284 |
-
d["geo_confidence"] = float(d.get("geo_confidence") or 0.0)
|
| 285 |
-
d["is_approximate"] = d.get("geo_precision", 3) >= 3
|
| 286 |
-
d["ai_analysis"] = d.get("ai_analysis")
|
| 287 |
-
if d.get("geom"): del d["geom"]
|
| 288 |
|
| 289 |
return {
|
| 290 |
"status": 200,
|
|
|
|
| 7 |
|
| 8 |
router = APIRouter(prefix="/conflicts", tags=["Conflicts"])
|
| 9 |
|
| 10 |
+
def serialize_event(d: dict) -> dict:
|
| 11 |
+
"""Standardized event serialization for all API responses."""
|
| 12 |
+
d["event_time"] = d["event_time"].isoformat() + "Z" if d.get("event_time") else None
|
| 13 |
+
d["event_date"] = d["event_date"].isoformat() if d.get("event_date") else None
|
| 14 |
+
d["ingested_at"] = d["ingested_at"].isoformat() + "Z" if d.get("ingested_at") else None
|
| 15 |
+
d["geo_confidence"] = float(d.get("geo_confidence") or 0.0)
|
| 16 |
+
d["is_approximate"] = d.get("geo_precision", 3) >= 3
|
| 17 |
+
d["ai_analysis"] = d.get("ai_analysis")
|
| 18 |
+
d["verification_count"] = d.get("verification_count", 1)
|
| 19 |
+
d["source_urls"] = d.get("source_urls") or []
|
| 20 |
+
d["strategic_relevance"] = d.get("strategic_relevance", "LOW")
|
| 21 |
+
if d.get("geom"): del d["geom"]
|
| 22 |
+
return d
|
| 23 |
+
|
| 24 |
async def check_cache(request: Request, cache_key: str):
|
| 25 |
if db.redis:
|
| 26 |
cached = await db.redis.get(cache_key)
|
|
|
|
| 90 |
|
| 91 |
data = [dict(r) for r in records]
|
| 92 |
for d in data:
|
| 93 |
+
serialize_event(d)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 94 |
|
| 95 |
response = {
|
| 96 |
"status": 200,
|
|
|
|
| 123 |
|
| 124 |
data = [dict(r) for r in records]
|
| 125 |
for d in data:
|
| 126 |
+
serialize_event(d)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 127 |
|
| 128 |
response = {
|
| 129 |
"status": 200,
|
|
|
|
| 152 |
|
| 153 |
data = [dict(r) for r in records]
|
| 154 |
for d in data:
|
| 155 |
+
serialize_event(d)
|
|
|
|
|
|
|
|
|
|
| 156 |
|
| 157 |
res = {"status": 200, "success": True, "count": len(data), "data": data}
|
| 158 |
await set_cache(cache_key, res, ttl=60) # Short TTL for live data
|
|
|
|
| 178 |
|
| 179 |
data = [dict(r) for r in records]
|
| 180 |
for d in data:
|
| 181 |
+
serialize_event(d)
|
|
|
|
|
|
|
|
|
|
| 182 |
|
| 183 |
res = {"status": 200, "success": True, "count": len(data), "data": data}
|
| 184 |
await set_cache(cache_key, res, ttl=3600) # Long TTL for history
|
|
|
|
| 212 |
|
| 213 |
data = [dict(r) for r in records]
|
| 214 |
for d in data:
|
| 215 |
+
serialize_event(d)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 216 |
|
| 217 |
response = {
|
| 218 |
"status": 200,
|
|
|
|
| 240 |
|
| 241 |
data = [dict(r) for r in records]
|
| 242 |
for d in data:
|
| 243 |
+
serialize_event(d)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 244 |
|
| 245 |
response = {
|
| 246 |
"status": 200,
|
|
|
|
| 262 |
raise HTTPException(status_code=404, detail="Event not found")
|
| 263 |
|
| 264 |
d = dict(record)
|
| 265 |
+
serialize_event(d)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 266 |
|
| 267 |
return {
|
| 268 |
"status": 200,
|
frontend/src/Dashboard.jsx
CHANGED
|
@@ -216,10 +216,6 @@ const Dashboard = () => {
|
|
| 216 |
<IncidentIntelligenceCard
|
| 217 |
event={activeIntelEvent}
|
| 218 |
onClose={() => setActiveIntelEvent(null)}
|
| 219 |
-
onRequestLiveAnalysis={(ev) => {
|
| 220 |
-
// Optional: trigger manual live re-analysis if ever needed
|
| 221 |
-
console.log("Live analysis requested for", ev.event_id);
|
| 222 |
-
}}
|
| 223 |
/>
|
| 224 |
)}
|
| 225 |
|
|
|
|
| 216 |
<IncidentIntelligenceCard
|
| 217 |
event={activeIntelEvent}
|
| 218 |
onClose={() => setActiveIntelEvent(null)}
|
|
|
|
|
|
|
|
|
|
|
|
|
| 219 |
/>
|
| 220 |
)}
|
| 221 |
|
frontend/src/components/IncidentIntelligenceCard.jsx
CHANGED
|
@@ -1,14 +1,28 @@
|
|
| 1 |
-
import React
|
| 2 |
import {
|
| 3 |
X, ShieldAlert, Target, Users, MapPin, ExternalLink,
|
| 4 |
-
Clock, Zap,
|
|
|
|
| 5 |
} from 'lucide-react';
|
| 6 |
|
| 7 |
-
const IncidentIntelligenceCard = ({ event, onClose
|
| 8 |
-
const [activeTab, setActiveTab] = useState('sitrep');
|
| 9 |
-
|
| 10 |
if (!event) return null;
|
| 11 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 12 |
return (
|
| 13 |
<div className="incident-intel-card glass-panel shadow-tactical animate-fade-in" style={{
|
| 14 |
position: 'absolute',
|
|
@@ -16,29 +30,42 @@ const IncidentIntelligenceCard = ({ event, onClose, onRequestLiveAnalysis }) =>
|
|
| 16 |
left: '20px',
|
| 17 |
width: '420px',
|
| 18 |
maxHeight: 'calc(100vh - 120px)',
|
| 19 |
-
background: 'rgba(5, 7, 12, 0.
|
| 20 |
borderRadius: '12px',
|
| 21 |
-
border:
|
| 22 |
zIndex: 1000,
|
| 23 |
display: 'flex',
|
| 24 |
flexDirection: 'column',
|
| 25 |
overflow: 'hidden',
|
| 26 |
-
color: 'var(--text-primary)'
|
|
|
|
| 27 |
}}>
|
| 28 |
{/* HEADER */}
|
| 29 |
<div style={{
|
| 30 |
padding: '15px 20px',
|
| 31 |
-
background: 'rgba(255,255,255,0.03)',
|
| 32 |
borderBottom: '1px solid var(--border-glass)',
|
| 33 |
display: 'flex',
|
| 34 |
justifyContent: 'space-between',
|
| 35 |
alignItems: 'center'
|
| 36 |
}}>
|
| 37 |
-
<div style={{ display: 'flex', alignItems: 'center', gap: '
|
| 38 |
-
|
| 39 |
-
|
| 40 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 41 |
</span>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 42 |
</div>
|
| 43 |
<button
|
| 44 |
onClick={onClose}
|
|
@@ -50,100 +77,79 @@ const IncidentIntelligenceCard = ({ event, onClose, onRequestLiveAnalysis }) =>
|
|
| 50 |
|
| 51 |
{/* EVENT CORE INFO */}
|
| 52 |
<div style={{ padding: '20px' }}>
|
| 53 |
-
<h2 style={{ fontSize: '
|
| 54 |
{event.title}
|
| 55 |
</h2>
|
| 56 |
-
|
| 57 |
-
|
| 58 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 59 |
</div>
|
| 60 |
-
<div style={{ display: 'flex', alignItems: 'center', gap: '
|
| 61 |
-
<
|
| 62 |
</div>
|
| 63 |
-
<a
|
| 64 |
-
href={event.source_url} target="_blank" rel="noreferrer"
|
| 65 |
-
style={{ display: 'flex', alignItems: 'center', gap: '6px', fontSize: '11px', color: 'var(--accent-cyan)', textDecoration: 'none' }}
|
| 66 |
-
>
|
| 67 |
-
<ExternalLink size={12} /> VERIFY SOURCE
|
| 68 |
-
</a>
|
| 69 |
</div>
|
| 70 |
|
| 71 |
-
{/*
|
| 72 |
-
<div style={{ display: '
|
| 73 |
-
<
|
| 74 |
-
<
|
|
|
|
|
|
|
|
|
|
|
|
|
| 75 |
</div>
|
| 76 |
-
|
| 77 |
-
{/*
|
| 78 |
-
{
|
| 79 |
-
<div
|
| 80 |
-
{
|
| 81 |
-
|
| 82 |
-
|
| 83 |
-
|
| 84 |
-
) : (
|
| 85 |
-
<div style={{ textAlign: 'center', padding: '30px 0', opacity: 0.5 }}>
|
| 86 |
-
<Activity size={32} style={{ marginBottom: '15px' }} />
|
| 87 |
-
<div style={{ fontSize: '11px', fontWeight: 800 }}>NO PRE-STORED ANALYSIS FOUND</div>
|
| 88 |
-
<button
|
| 89 |
-
onClick={() => onRequestLiveAnalysis(event)}
|
| 90 |
-
style={{
|
| 91 |
-
marginTop: '20px', padding: '10px 20px', background: 'var(--accent-cyan)',
|
| 92 |
-
color: 'black', border: 'none', borderRadius: '4px', fontSize: '10px', fontWeight: 900,
|
| 93 |
-
cursor: 'pointer'
|
| 94 |
-
}}
|
| 95 |
-
>
|
| 96 |
-
REQUEST LIVE SITREP
|
| 97 |
-
</button>
|
| 98 |
-
</div>
|
| 99 |
-
)}
|
| 100 |
</div>
|
| 101 |
)}
|
| 102 |
|
| 103 |
-
{/*
|
| 104 |
-
{
|
| 105 |
-
<div
|
| 106 |
-
<div style={{
|
| 107 |
-
|
| 108 |
-
<DetailBox label="FATALITIES" value={event.fatalities} color="var(--accent-red)" icon={<Target size={14}/>} />
|
| 109 |
-
<DetailBox label="EQP / WEAPON" value={event.weapon || 'KINETIC'} icon={<Zap size={14}/>} />
|
| 110 |
-
<DetailBox label="ACCURACY" value={`${Math.round(event.geo_confidence * 100)}%`} color="var(--accent-cyan)" />
|
| 111 |
-
</div>
|
| 112 |
-
|
| 113 |
-
<div className="glass-panel" style={{ padding: '15px', borderRadius: '8px', background: 'rgba(0,0,0,0.2)' }}>
|
| 114 |
-
<div style={{ fontSize: '8px', fontWeight: 900, color: 'var(--text-dim)', letterSpacing: '1px', marginBottom: '8px' }}>TACTICAL NOTES</div>
|
| 115 |
-
<div style={{ fontSize: '12px', lineHeight: '1.6', color: 'var(--text-secondary)' }}>
|
| 116 |
-
{event.notes || 'No specialized tactical descriptors extracted for this event.'}
|
| 117 |
-
</div>
|
| 118 |
</div>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 119 |
</div>
|
| 120 |
)}
|
| 121 |
</div>
|
| 122 |
|
| 123 |
-
{/* FOOTER
|
| 124 |
<div style={{
|
| 125 |
-
padding: '
|
| 126 |
display: 'flex', justifyContent: 'space-between', alignItems: 'center'
|
| 127 |
}}>
|
| 128 |
<div style={{ display: 'flex', alignItems: 'center', gap: '6px' }}>
|
| 129 |
-
<div style={{ width: '6px', height: '6px', borderRadius: '50%', background: 'var(--accent-green)' }}></div>
|
| 130 |
-
<span style={{ fontSize: '9px', fontWeight: 900, color: 'var(--text-dim)' }}>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 131 |
</div>
|
| 132 |
-
{event.ai_analysis && (
|
| 133 |
-
<button
|
| 134 |
-
onClick={() => onRequestLiveAnalysis(event)}
|
| 135 |
-
style={{
|
| 136 |
-
background: 'none', border: 'none', color: 'var(--accent-cyan)', fontSize: '9px', fontWeight: 900,
|
| 137 |
-
cursor: 'pointer', display: 'flex', alignItems: 'center', gap: '4px'
|
| 138 |
-
}}
|
| 139 |
-
>
|
| 140 |
-
REFRESH ANALYSIS <ChevronRight size={10} />
|
| 141 |
-
</button>
|
| 142 |
-
)}
|
| 143 |
</div>
|
| 144 |
|
| 145 |
<style dangerouslySetInnerHTML={{ __html: `
|
| 146 |
-
.tab-content { transition: all 0.3s ease; }
|
| 147 |
@keyframes fade-in { from { opacity: 0; transform: translateY(10px); } to { opacity: 1; transform: translateY(0); } }
|
| 148 |
.animate-fade-in { animation: fade-in 0.3s ease-out; }
|
| 149 |
`}} />
|
|
@@ -151,30 +157,13 @@ const IncidentIntelligenceCard = ({ event, onClose, onRequestLiveAnalysis }) =>
|
|
| 151 |
);
|
| 152 |
};
|
| 153 |
|
| 154 |
-
const TabItem = ({ active, label, onClick }) => (
|
| 155 |
-
<div
|
| 156 |
-
onClick={onClick}
|
| 157 |
-
style={{
|
| 158 |
-
padding: '10px 15px',
|
| 159 |
-
fontSize: '11px',
|
| 160 |
-
fontWeight: 900,
|
| 161 |
-
cursor: 'pointer',
|
| 162 |
-
color: active ? 'white' : 'var(--text-dim)',
|
| 163 |
-
borderBottom: active ? '2px solid var(--accent-cyan)' : '2px solid transparent',
|
| 164 |
-
transition: 'all 0.2s'
|
| 165 |
-
}}
|
| 166 |
-
>
|
| 167 |
-
{label}
|
| 168 |
-
</div>
|
| 169 |
-
);
|
| 170 |
-
|
| 171 |
const DetailBox = ({ label, value, color, icon }) => (
|
| 172 |
<div style={{ background: 'rgba(255,255,255,0.02)', padding: '12px', borderRadius: '8px', border: '1px solid var(--border-glass)' }}>
|
| 173 |
<div style={{ fontSize: '8px', fontWeight: 900, color: 'var(--text-dim)', letterSpacing: '1px', marginBottom: '6px', display: 'flex', alignItems: 'center', gap: '6px' }}>
|
| 174 |
{icon} {label}
|
| 175 |
</div>
|
| 176 |
-
<div style={{ fontSize: '
|
| 177 |
-
{value}
|
| 178 |
</div>
|
| 179 |
</div>
|
| 180 |
);
|
|
|
|
| 1 |
+
import React from 'react';
|
| 2 |
import {
|
| 3 |
X, ShieldAlert, Target, Users, MapPin, ExternalLink,
|
| 4 |
+
Clock, Zap, ChevronRight, AlertTriangle,
|
| 5 |
+
CheckCircle
|
| 6 |
} from 'lucide-react';
|
| 7 |
|
| 8 |
+
const IncidentIntelligenceCard = ({ event, onClose }) => {
|
|
|
|
|
|
|
| 9 |
if (!event) return null;
|
| 10 |
|
| 11 |
+
const isVerified = (event.verification_count || 1) > 1;
|
| 12 |
+
const relevance = event.strategic_relevance || 'LOW';
|
| 13 |
+
const relevanceColors = { CRITICAL: 'var(--accent-red)', HIGH: 'var(--accent-amber)', MEDIUM: 'var(--accent-cyan)', LOW: 'var(--text-dim)' };
|
| 14 |
+
|
| 15 |
+
// Safe URL parsing
|
| 16 |
+
const getSourceUrls = () => {
|
| 17 |
+
const urls = event.source_urls?.length ? event.source_urls : [event.source_url];
|
| 18 |
+
return urls.filter(u => u && u.startsWith('http'));
|
| 19 |
+
};
|
| 20 |
+
|
| 21 |
+
const safeDomain = (url) => {
|
| 22 |
+
try { return new URL(url).hostname.replace('www.', ''); }
|
| 23 |
+
catch { return 'source'; }
|
| 24 |
+
};
|
| 25 |
+
|
| 26 |
return (
|
| 27 |
<div className="incident-intel-card glass-panel shadow-tactical animate-fade-in" style={{
|
| 28 |
position: 'absolute',
|
|
|
|
| 30 |
left: '20px',
|
| 31 |
width: '420px',
|
| 32 |
maxHeight: 'calc(100vh - 120px)',
|
| 33 |
+
background: 'rgba(5, 7, 12, 0.97)',
|
| 34 |
borderRadius: '12px',
|
| 35 |
+
border: `1px solid ${isVerified ? 'rgba(16, 185, 129, 0.3)' : 'var(--border-glass)'}`,
|
| 36 |
zIndex: 1000,
|
| 37 |
display: 'flex',
|
| 38 |
flexDirection: 'column',
|
| 39 |
overflow: 'hidden',
|
| 40 |
+
color: 'var(--text-primary)',
|
| 41 |
+
overflowY: 'auto'
|
| 42 |
}}>
|
| 43 |
{/* HEADER */}
|
| 44 |
<div style={{
|
| 45 |
padding: '15px 20px',
|
| 46 |
+
background: isVerified ? 'rgba(16, 185, 129, 0.05)' : 'rgba(255,255,255,0.03)',
|
| 47 |
borderBottom: '1px solid var(--border-glass)',
|
| 48 |
display: 'flex',
|
| 49 |
justifyContent: 'space-between',
|
| 50 |
alignItems: 'center'
|
| 51 |
}}>
|
| 52 |
+
<div style={{ display: 'flex', alignItems: 'center', gap: '8px', flexWrap: 'wrap' }}>
|
| 53 |
+
{isVerified ? (
|
| 54 |
+
<CheckCircle size={14} color="var(--accent-green)" />
|
| 55 |
+
) : (
|
| 56 |
+
<ShieldAlert size={14} color="var(--accent-red)" />
|
| 57 |
+
)}
|
| 58 |
+
<span style={{ fontSize: '10px', fontWeight: 900, letterSpacing: '2px', color: isVerified ? 'var(--accent-green)' : 'var(--text-dim)' }}>
|
| 59 |
+
{isVerified ? `VERIFIED × ${event.verification_count} SOURCES` : `INTEL CASE: ${event.event_id?.slice(0, 8)}`}
|
| 60 |
</span>
|
| 61 |
+
<div style={{
|
| 62 |
+
background: `${relevanceColors[relevance]}20`,
|
| 63 |
+
color: relevanceColors[relevance],
|
| 64 |
+
padding: '2px 6px', borderRadius: '4px', fontSize: '8px', fontWeight: 900,
|
| 65 |
+
border: `1px solid ${relevanceColors[relevance]}40`
|
| 66 |
+
}}>
|
| 67 |
+
{relevance}
|
| 68 |
+
</div>
|
| 69 |
</div>
|
| 70 |
<button
|
| 71 |
onClick={onClose}
|
|
|
|
| 77 |
|
| 78 |
{/* EVENT CORE INFO */}
|
| 79 |
<div style={{ padding: '20px' }}>
|
| 80 |
+
<h2 style={{ fontSize: '16px', fontWeight: 900, margin: '0 0 12px 0', lineHeight: '1.35' }}>
|
| 81 |
{event.title}
|
| 82 |
</h2>
|
| 83 |
+
|
| 84 |
+
{/* META ROW */}
|
| 85 |
+
<div style={{ display: 'flex', flexWrap: 'wrap', gap: '10px', marginBottom: '20px' }}>
|
| 86 |
+
<div style={{ display: 'flex', alignItems: 'center', gap: '5px', fontSize: '11px', color: 'var(--text-secondary)' }}>
|
| 87 |
+
<MapPin size={12} /> {event.location_raw || event.city || 'Sector'}, {event.country}
|
| 88 |
+
</div>
|
| 89 |
+
<div style={{ display: 'flex', alignItems: 'center', gap: '5px', fontSize: '11px', color: 'var(--text-secondary)' }}>
|
| 90 |
+
<Clock size={12} /> {event.event_time ? new Date(event.event_time).toLocaleString() : 'Unknown'}
|
| 91 |
</div>
|
| 92 |
+
<div style={{ display: 'flex', alignItems: 'center', gap: '5px', fontSize: '11px', color: 'var(--text-secondary)' }}>
|
| 93 |
+
<Zap size={12} /> {event.event_type || 'Unknown'}
|
| 94 |
</div>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 95 |
</div>
|
| 96 |
|
| 97 |
+
{/* ENTITY GRID */}
|
| 98 |
+
<div style={{ display: 'grid', gridTemplateColumns: '1fr 1fr', gap: '10px', marginBottom: '20px' }}>
|
| 99 |
+
<DetailBox label="PRIMARY ACTOR" value={event.actor1 || 'UNKNOWN'} icon={<Users size={14}/>} />
|
| 100 |
+
<DetailBox label="SECONDARY ACTOR" value={event.actor2 || 'N/A'} icon={<Target size={14}/>} />
|
| 101 |
+
<DetailBox label="FATALITIES" value={event.fatalities || 0} color="var(--accent-red)" icon={<AlertTriangle size={14}/>} />
|
| 102 |
+
<DetailBox label="WEAPON SYSTEM" value={event.weapon || 'NOT SPECIFIED'} icon={<Zap size={14}/>} />
|
| 103 |
+
<DetailBox label="GEO ACCURACY" value={`${Math.round((event.geo_confidence || 0) * 100)}%`} color="var(--accent-cyan)" />
|
| 104 |
+
<DetailBox label="PRECISION" value={event.geo_precision === 1 ? 'EXACT' : event.geo_precision === 2 ? 'ADMIN' : 'BROAD'} color={event.geo_precision === 1 ? 'var(--accent-green)' : 'var(--accent-amber)'} />
|
| 105 |
</div>
|
| 106 |
+
|
| 107 |
+
{/* TACTICAL NOTES */}
|
| 108 |
+
{event.notes && (
|
| 109 |
+
<div style={{ padding: '12px', background: 'rgba(0,0,0,0.2)', borderRadius: '8px', border: '1px solid var(--border-glass)', marginBottom: '20px' }}>
|
| 110 |
+
<div style={{ fontSize: '8px', fontWeight: 900, color: 'var(--text-dim)', letterSpacing: '1px', marginBottom: '6px' }}>TACTICAL NOTES</div>
|
| 111 |
+
<div style={{ fontSize: '12px', lineHeight: '1.6', color: 'var(--text-secondary)' }}>
|
| 112 |
+
{event.notes}
|
| 113 |
+
</div>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 114 |
</div>
|
| 115 |
)}
|
| 116 |
|
| 117 |
+
{/* EVIDENCE TRAIN */}
|
| 118 |
+
{getSourceUrls().length > 0 && (
|
| 119 |
+
<div style={{ display: 'flex', flexDirection: 'column', gap: '6px', padding: '12px', background: 'rgba(0,0,0,0.3)', borderRadius: '8px', border: '1px solid var(--border-glass)' }}>
|
| 120 |
+
<div style={{ fontSize: '8px', fontWeight: 900, color: isVerified ? 'var(--accent-green)' : 'var(--text-dim)', letterSpacing: '1px' }}>
|
| 121 |
+
{isVerified ? '✓ MULTI-SOURCE VERIFIED' : 'SOURCE'}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 122 |
</div>
|
| 123 |
+
{getSourceUrls().map((url, idx) => (
|
| 124 |
+
<a
|
| 125 |
+
key={idx}
|
| 126 |
+
href={url} target="_blank" rel="noreferrer"
|
| 127 |
+
style={{ display: 'flex', alignItems: 'center', gap: '8px', fontSize: '10px', color: 'var(--accent-cyan)', textDecoration: 'none', opacity: 0.85 }}
|
| 128 |
+
>
|
| 129 |
+
<ExternalLink size={10} /> SOURCE {idx + 1}: {safeDomain(url)}
|
| 130 |
+
</a>
|
| 131 |
+
))}
|
| 132 |
</div>
|
| 133 |
)}
|
| 134 |
</div>
|
| 135 |
|
| 136 |
+
{/* FOOTER */}
|
| 137 |
<div style={{
|
| 138 |
+
padding: '12px 20px', background: 'rgba(0,0,0,0.3)', borderTop: '1px solid var(--border-glass)',
|
| 139 |
display: 'flex', justifyContent: 'space-between', alignItems: 'center'
|
| 140 |
}}>
|
| 141 |
<div style={{ display: 'flex', alignItems: 'center', gap: '6px' }}>
|
| 142 |
+
<div style={{ width: '6px', height: '6px', borderRadius: '50%', background: event.ai_classified !== false ? 'var(--accent-green)' : 'var(--accent-amber)' }}></div>
|
| 143 |
+
<span style={{ fontSize: '9px', fontWeight: 900, color: 'var(--text-dim)' }}>
|
| 144 |
+
{event.ai_classified !== false ? 'AI CLASSIFIED' : 'REGEX FALLBACK'}
|
| 145 |
+
</span>
|
| 146 |
+
</div>
|
| 147 |
+
<div style={{ fontSize: '9px', fontWeight: 800, color: 'var(--text-dim)', fontFamily: 'var(--font-mono)' }}>
|
| 148 |
+
{event.source || 'SRC'} // {event.event_id?.slice(0, 12)}
|
| 149 |
</div>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 150 |
</div>
|
| 151 |
|
| 152 |
<style dangerouslySetInnerHTML={{ __html: `
|
|
|
|
| 153 |
@keyframes fade-in { from { opacity: 0; transform: translateY(10px); } to { opacity: 1; transform: translateY(0); } }
|
| 154 |
.animate-fade-in { animation: fade-in 0.3s ease-out; }
|
| 155 |
`}} />
|
|
|
|
| 157 |
);
|
| 158 |
};
|
| 159 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 160 |
const DetailBox = ({ label, value, color, icon }) => (
|
| 161 |
<div style={{ background: 'rgba(255,255,255,0.02)', padding: '12px', borderRadius: '8px', border: '1px solid var(--border-glass)' }}>
|
| 162 |
<div style={{ fontSize: '8px', fontWeight: 900, color: 'var(--text-dim)', letterSpacing: '1px', marginBottom: '6px', display: 'flex', alignItems: 'center', gap: '6px' }}>
|
| 163 |
{icon} {label}
|
| 164 |
</div>
|
| 165 |
+
<div style={{ fontSize: '12px', fontWeight: 900, color: color || 'white', textTransform: 'uppercase', wordBreak: 'break-word' }}>
|
| 166 |
+
{value ?? 'N/A'}
|
| 167 |
</div>
|
| 168 |
</div>
|
| 169 |
);
|
frontend/src/components/TacticalFeed.jsx
CHANGED
|
@@ -69,6 +69,16 @@ const TacticalFeed = ({ events, selectedId, onSelectEvent }) => {
|
|
| 69 |
<span style={{ fontSize: '9px', fontWeight: 900, color: ev.severity_score >= 8 ? 'var(--accent-red)' : 'var(--text-secondary)', letterSpacing: '1px' }}>
|
| 70 |
{ev.event_type?.toUpperCase() || 'GENERAL'}
|
| 71 |
</span>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 72 |
</div>
|
| 73 |
<span style={{ fontSize: '9px', color: 'var(--text-dim)', fontWeight: 700, fontFamily: 'var(--font-mono)' }}>
|
| 74 |
{getTimeAgo(ev.event_time)}
|
|
|
|
| 69 |
<span style={{ fontSize: '9px', fontWeight: 900, color: ev.severity_score >= 8 ? 'var(--accent-red)' : 'var(--text-secondary)', letterSpacing: '1px' }}>
|
| 70 |
{ev.event_type?.toUpperCase() || 'GENERAL'}
|
| 71 |
</span>
|
| 72 |
+
{(ev.verification_count || 1) > 1 && (
|
| 73 |
+
<span style={{ fontSize: '8px', fontWeight: 900, color: 'var(--accent-green)', background: 'rgba(16,185,129,0.1)', padding: '1px 5px', borderRadius: '3px', border: '1px solid rgba(16,185,129,0.3)' }}>
|
| 74 |
+
✓ {ev.verification_count}
|
| 75 |
+
</span>
|
| 76 |
+
)}
|
| 77 |
+
{(ev.strategic_relevance === 'CRITICAL' || ev.strategic_relevance === 'HIGH') && (
|
| 78 |
+
<span style={{ fontSize: '8px', fontWeight: 900, color: ev.strategic_relevance === 'CRITICAL' ? 'var(--accent-red)' : 'var(--accent-amber)', background: ev.strategic_relevance === 'CRITICAL' ? 'rgba(244,63,94,0.1)' : 'rgba(245,158,11,0.1)', padding: '1px 5px', borderRadius: '3px', border: `1px solid ${ev.strategic_relevance === 'CRITICAL' ? 'rgba(244,63,94,0.3)' : 'rgba(245,158,11,0.3)'}` }}>
|
| 79 |
+
{ev.strategic_relevance}
|
| 80 |
+
</span>
|
| 81 |
+
)}
|
| 82 |
</div>
|
| 83 |
<span style={{ fontSize: '9px', color: 'var(--text-dim)', fontWeight: 700, fontFamily: 'var(--font-mono)' }}>
|
| 84 |
{getTimeAgo(ev.event_time)}
|
frontend/src/components/TacticalMap.jsx
CHANGED
|
@@ -1,5 +1,5 @@
|
|
| 1 |
import React, { useState, useEffect, useMemo } from 'react';
|
| 2 |
-
import { MapContainer, TileLayer, Marker, Popup, Circle, useMap } from 'react-leaflet';
|
| 3 |
import L from 'leaflet';
|
| 4 |
import 'leaflet/dist/leaflet.css';
|
| 5 |
import { Target, Shield, Zap, MapPin, Activity, AlertTriangle, Users } from 'lucide-react';
|
|
@@ -128,6 +128,30 @@ const TacticalMap = ({ events, layerData, selectedEvent, onDeepAnalyze, layers }
|
|
| 128 |
</Circle>
|
| 129 |
))}
|
| 130 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 131 |
{/* CIVILIAN RISK VECTOR */}
|
| 132 |
{layers.civilians.active && (events || []).filter(e => e.fatalities_civilians > 0).map(ev => (
|
| 133 |
<Marker key={`civ-${ev.event_id}`} position={[ev.lat, ev.lon]} icon={createCivilianIcon()}>
|
|
@@ -149,6 +173,16 @@ const TacticalMap = ({ events, layerData, selectedEvent, onDeepAnalyze, layers }
|
|
| 149 |
|
| 150 |
return (
|
| 151 |
<React.Fragment key={ev.event_id}>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 152 |
{/* Uncertainty Zone for non-exact points */}
|
| 153 |
{!isExact && (
|
| 154 |
<Circle
|
|
@@ -170,47 +204,117 @@ const TacticalMap = ({ events, layerData, selectedEvent, onDeepAnalyze, layers }
|
|
| 170 |
zIndexOffset={isExact ? 1000 : 500}
|
| 171 |
>
|
| 172 |
<Popup className="tactical-popup">
|
| 173 |
-
<div
|
| 174 |
-
|
| 175 |
-
|
| 176 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 177 |
</div>
|
| 178 |
-
|
| 179 |
-
|
| 180 |
-
|
| 181 |
-
|
| 182 |
-
|
| 183 |
-
|
| 184 |
-
|
| 185 |
-
|
| 186 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 187 |
</div>
|
|
|
|
| 188 |
|
| 189 |
-
|
| 190 |
-
|
| 191 |
-
|
| 192 |
-
|
| 193 |
-
<
|
| 194 |
-
|
| 195 |
-
|
| 196 |
-
|
|
|
|
|
|
|
|
|
|
| 197 |
</div>
|
| 198 |
-
|
| 199 |
-
|
| 200 |
-
|
| 201 |
-
|
| 202 |
-
|
| 203 |
-
|
| 204 |
-
|
| 205 |
-
|
| 206 |
-
|
| 207 |
-
|
| 208 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 209 |
</div>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 210 |
|
| 211 |
-
|
| 212 |
-
|
| 213 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 214 |
</div>
|
| 215 |
</div>
|
| 216 |
</Popup>
|
|
|
|
| 1 |
import React, { useState, useEffect, useMemo } from 'react';
|
| 2 |
+
import { MapContainer, TileLayer, Marker, Popup, Circle, Polyline, useMap } from 'react-leaflet';
|
| 3 |
import L from 'leaflet';
|
| 4 |
import 'leaflet/dist/leaflet.css';
|
| 5 |
import { Target, Shield, Zap, MapPin, Activity, AlertTriangle, Users } from 'lucide-react';
|
|
|
|
| 128 |
</Circle>
|
| 129 |
))}
|
| 130 |
|
| 131 |
+
{/* STRATEGIC TENSION VECTORS (Frontier Intelligence) */}
|
| 132 |
+
{useMemo(() => {
|
| 133 |
+
const highIntensity = (events || []).filter(e => e.severity_score > 7.5);
|
| 134 |
+
const vectors = [];
|
| 135 |
+
for (let i = 0; i < highIntensity.length; i++) {
|
| 136 |
+
for (let j = i + 1; j < highIntensity.length; j++) {
|
| 137 |
+
const e1 = highIntensity[i];
|
| 138 |
+
const e2 = highIntensity[j];
|
| 139 |
+
// Connect if same country or same actor (Strategic Handshake)
|
| 140 |
+
if (e1.country_iso3 === e2.country_iso3 || (e1.actor1 && e1.actor1 === e2.actor1)) {
|
| 141 |
+
vectors.push({ id: `${e1.id}-${e2.id}`, coords: [[e1.lat, e1.lon], [e2.lat, e2.lon]] });
|
| 142 |
+
}
|
| 143 |
+
}
|
| 144 |
+
}
|
| 145 |
+
return vectors.slice(0, 8); // Performance limit
|
| 146 |
+
}, [events]).map(v => (
|
| 147 |
+
<Polyline
|
| 148 |
+
key={`vector-${v.id}`}
|
| 149 |
+
positions={v.coords}
|
| 150 |
+
className="tension-vector"
|
| 151 |
+
pathOptions={{ color: 'var(--accent-red)', weight: 1, opacity: 0.3 }}
|
| 152 |
+
/>
|
| 153 |
+
))}
|
| 154 |
+
|
| 155 |
{/* CIVILIAN RISK VECTOR */}
|
| 156 |
{layers.civilians.active && (events || []).filter(e => e.fatalities_civilians > 0).map(ev => (
|
| 157 |
<Marker key={`civ-${ev.event_id}`} position={[ev.lat, ev.lon]} icon={createCivilianIcon()}>
|
|
|
|
| 173 |
|
| 174 |
return (
|
| 175 |
<React.Fragment key={ev.event_id}>
|
| 176 |
+
{/* Verified Incident Halo (Frontier Intelligence) */}
|
| 177 |
+
{ev.verification_count > 1 && (
|
| 178 |
+
<Circle
|
| 179 |
+
center={[ev.lat, ev.lon]}
|
| 180 |
+
radius={8000}
|
| 181 |
+
className="verified-pulse"
|
| 182 |
+
pathOptions={{ color: 'var(--accent-green)', fillColor: 'var(--accent-green)', fillOpacity: 0.2, weight: 1 }}
|
| 183 |
+
/>
|
| 184 |
+
)}
|
| 185 |
+
|
| 186 |
{/* Uncertainty Zone for non-exact points */}
|
| 187 |
{!isExact && (
|
| 188 |
<Circle
|
|
|
|
| 204 |
zIndexOffset={isExact ? 1000 : 500}
|
| 205 |
>
|
| 206 |
<Popup className="tactical-popup">
|
| 207 |
+
<div style={{ minWidth: '340px', maxWidth: '380px' }}>
|
| 208 |
+
{/* THREAT BANNER */}
|
| 209 |
+
<div style={{
|
| 210 |
+
padding: '12px 16px',
|
| 211 |
+
background: `linear-gradient(135deg, ${eventColor}15, transparent)`,
|
| 212 |
+
borderBottom: `1px solid ${eventColor}30`,
|
| 213 |
+
display: 'flex', justifyContent: 'space-between', alignItems: 'center'
|
| 214 |
+
}}>
|
| 215 |
+
<div style={{ display: 'flex', alignItems: 'center', gap: '8px' }}>
|
| 216 |
+
<div style={{ width: '8px', height: '8px', borderRadius: '50%', background: eventColor, boxShadow: `0 0 8px ${eventColor}` }}></div>
|
| 217 |
+
<span style={{ fontSize: '9px', fontWeight: 900, letterSpacing: '2px', color: eventColor }}>
|
| 218 |
+
{ev.event_type?.toUpperCase() || 'ENGAGEMENT'}
|
| 219 |
+
</span>
|
| 220 |
</div>
|
| 221 |
+
<div style={{ display: 'flex', gap: '5px', alignItems: 'center' }}>
|
| 222 |
+
{(ev.verification_count || 1) > 1 && (
|
| 223 |
+
<span style={{ fontSize: '8px', fontWeight: 900, color: '#10b981', background: 'rgba(16,185,129,0.12)', padding: '2px 7px', borderRadius: '3px', border: '1px solid rgba(16,185,129,0.25)' }}>
|
| 224 |
+
✓ {ev.verification_count}
|
| 225 |
+
</span>
|
| 226 |
+
)}
|
| 227 |
+
<span style={{
|
| 228 |
+
fontSize: '8px', fontWeight: 900, padding: '2px 7px', borderRadius: '3px',
|
| 229 |
+
color: isExact ? '#06b6d4' : '#64748b',
|
| 230 |
+
border: `1px solid ${isExact ? 'rgba(6,182,212,0.3)' : 'rgba(255,255,255,0.08)'}`,
|
| 231 |
+
background: isExact ? 'rgba(6,182,212,0.08)' : 'rgba(255,255,255,0.03)'
|
| 232 |
+
}}>
|
| 233 |
+
{ev.geo_precision === 1 ? 'EXACT' : ev.geo_precision === 2 ? 'ADMIN' : 'APPROX'}
|
| 234 |
+
</span>
|
| 235 |
</div>
|
| 236 |
+
</div>
|
| 237 |
|
| 238 |
+
{/* TITLE + LOCATION */}
|
| 239 |
+
<div style={{ padding: '14px 16px 10px' }}>
|
| 240 |
+
<div style={{ fontSize: '14px', fontWeight: 800, color: 'white', lineHeight: '1.35', marginBottom: '8px' }}>
|
| 241 |
+
{ev.title}
|
| 242 |
+
</div>
|
| 243 |
+
<div style={{ display: 'flex', alignItems: 'center', gap: '5px', fontSize: '10px', color: '#94a3b8' }}>
|
| 244 |
+
<MapPin size={10} style={{ opacity: 0.5 }} />
|
| 245 |
+
{ev.location_raw || ev.admin1 || ev.country_iso3}
|
| 246 |
+
<span style={{ opacity: 0.3 }}>•</span>
|
| 247 |
+
{ev.country}
|
| 248 |
+
</div>
|
| 249 |
</div>
|
| 250 |
+
|
| 251 |
+
{/* SEVERITY + STATS ROW */}
|
| 252 |
+
<div style={{ padding: '0 16px 14px', display: 'flex', gap: '8px' }}>
|
| 253 |
+
{/* Severity Gauge */}
|
| 254 |
+
<div style={{
|
| 255 |
+
flex: 1, padding: '10px', borderRadius: '8px',
|
| 256 |
+
background: `linear-gradient(135deg, ${eventColor}08, ${eventColor}03)`,
|
| 257 |
+
border: `1px solid ${eventColor}20`, textAlign: 'center'
|
| 258 |
+
}}>
|
| 259 |
+
<div style={{ fontSize: '22px', fontWeight: 900, color: eventColor, lineHeight: 1 }}>{ev.severity_score}</div>
|
| 260 |
+
<div style={{ fontSize: '7px', fontWeight: 800, color: '#64748b', letterSpacing: '1.5px', marginTop: '4px' }}>SEVERITY</div>
|
| 261 |
+
</div>
|
| 262 |
+
{/* Confidence */}
|
| 263 |
+
<div style={{
|
| 264 |
+
flex: 1, padding: '10px', borderRadius: '8px',
|
| 265 |
+
background: 'rgba(6,182,212,0.04)', border: '1px solid rgba(6,182,212,0.12)', textAlign: 'center'
|
| 266 |
+
}}>
|
| 267 |
+
<div style={{ fontSize: '22px', fontWeight: 900, color: '#06b6d4', lineHeight: 1 }}>{Math.round((ev.geo_confidence||0) * 100)}%</div>
|
| 268 |
+
<div style={{ fontSize: '7px', fontWeight: 800, color: '#64748b', letterSpacing: '1.5px', marginTop: '4px' }}>CONFIDENCE</div>
|
| 269 |
+
</div>
|
| 270 |
+
{/* Fatalities */}
|
| 271 |
+
{ev.fatalities > 0 && (
|
| 272 |
+
<div style={{
|
| 273 |
+
flex: 1, padding: '10px', borderRadius: '8px',
|
| 274 |
+
background: 'rgba(244,63,94,0.05)', border: '1px solid rgba(244,63,94,0.15)', textAlign: 'center'
|
| 275 |
+
}}>
|
| 276 |
+
<div style={{ fontSize: '22px', fontWeight: 900, color: '#f43f5e', lineHeight: 1 }}>{ev.fatalities}</div>
|
| 277 |
+
<div style={{ fontSize: '7px', fontWeight: 800, color: '#64748b', letterSpacing: '1.5px', marginTop: '4px' }}>KIA</div>
|
| 278 |
</div>
|
| 279 |
+
)}
|
| 280 |
+
</div>
|
| 281 |
+
|
| 282 |
+
{/* ENTITY PILLS */}
|
| 283 |
+
{(ev.actor1 || ev.weapon || ev.actor2) && (
|
| 284 |
+
<div style={{ padding: '0 16px 14px', display: 'flex', flexWrap: 'wrap', gap: '6px' }}>
|
| 285 |
+
{ev.actor1 && (
|
| 286 |
+
<div style={{ fontSize: '9px', fontWeight: 800, color: '#06b6d4', background: 'rgba(6,182,212,0.08)', padding: '4px 10px', borderRadius: '20px', border: '1px solid rgba(6,182,212,0.2)' }}>
|
| 287 |
+
⚔ {ev.actor1}
|
| 288 |
+
</div>
|
| 289 |
+
)}
|
| 290 |
+
{ev.actor2 && (
|
| 291 |
+
<div style={{ fontSize: '9px', fontWeight: 800, color: '#94a3b8', background: 'rgba(255,255,255,0.04)', padding: '4px 10px', borderRadius: '20px', border: '1px solid rgba(255,255,255,0.08)' }}>
|
| 292 |
+
⛨ {ev.actor2}
|
| 293 |
+
</div>
|
| 294 |
+
)}
|
| 295 |
+
{ev.weapon && (
|
| 296 |
+
<div style={{ fontSize: '9px', fontWeight: 800, color: '#fbbf24', background: 'rgba(251,191,36,0.08)', padding: '4px 10px', borderRadius: '20px', border: '1px solid rgba(251,191,36,0.2)' }}>
|
| 297 |
+
◈ {ev.weapon}
|
| 298 |
+
</div>
|
| 299 |
+
)}
|
| 300 |
+
</div>
|
| 301 |
+
)}
|
| 302 |
|
| 303 |
+
{/* FOOTER: ANALYZE + META */}
|
| 304 |
+
<div style={{
|
| 305 |
+
padding: '10px 16px', background: 'rgba(0,0,0,0.25)', borderTop: '1px solid rgba(255,255,255,0.04)',
|
| 306 |
+
display: 'flex', justifyContent: 'space-between', alignItems: 'center'
|
| 307 |
+
}}>
|
| 308 |
+
<button
|
| 309 |
+
className="nav-command-btn active"
|
| 310 |
+
style={{ padding: '6px 14px', fontSize: '9px' }}
|
| 311 |
+
onClick={() => onDeepAnalyze(ev)}
|
| 312 |
+
>
|
| 313 |
+
INTEL REPORT
|
| 314 |
+
</button>
|
| 315 |
+
<span style={{ fontSize: '8px', color: '#475569', fontFamily: 'var(--font-mono)' }}>
|
| 316 |
+
{ev.source || 'SRC'} • {ev.country_iso3}
|
| 317 |
+
</span>
|
| 318 |
</div>
|
| 319 |
</div>
|
| 320 |
</Popup>
|
frontend/src/index.css
CHANGED
|
@@ -310,3 +310,33 @@ body {
|
|
| 310 |
0%, 100% { opacity: 1; transform: scale(1); }
|
| 311 |
50% { opacity: 0.3; transform: scale(0.8); }
|
| 312 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 310 |
0%, 100% { opacity: 1; transform: scale(1); }
|
| 311 |
50% { opacity: 0.3; transform: scale(0.8); }
|
| 312 |
}
|
| 313 |
+
|
| 314 |
+
/* --- FRONTIER COMMAND VISUALS --- */
|
| 315 |
+
.tension-vector {
|
| 316 |
+
stroke-dasharray: 10, 10;
|
| 317 |
+
animation: dash-flow 20s linear infinite;
|
| 318 |
+
filter: drop-shadow(0 0 5px var(--accent-red));
|
| 319 |
+
}
|
| 320 |
+
|
| 321 |
+
@keyframes dash-flow {
|
| 322 |
+
to { stroke-dashoffset: -1000; }
|
| 323 |
+
}
|
| 324 |
+
|
| 325 |
+
.theater-haze {
|
| 326 |
+
filter: blur(10px);
|
| 327 |
+
animation: heat-haze 8s infinite alternate ease-in-out;
|
| 328 |
+
}
|
| 329 |
+
|
| 330 |
+
@keyframes heat-haze {
|
| 331 |
+
0% { transform: scale(1) skew(0deg); opacity: 0.3; }
|
| 332 |
+
100% { transform: scale(1.1) skew(2deg); opacity: 0.5; }
|
| 333 |
+
}
|
| 334 |
+
|
| 335 |
+
.verified-pulse {
|
| 336 |
+
animation: verified-glow 2s infinite alternate;
|
| 337 |
+
}
|
| 338 |
+
|
| 339 |
+
@keyframes verified-glow {
|
| 340 |
+
0% { filter: drop-shadow(0 0 2px var(--accent-green)); }
|
| 341 |
+
100% { filter: drop-shadow(0 0 12px var(--accent-green)); }
|
| 342 |
+
}
|
poller/classifier.py
CHANGED
|
@@ -153,27 +153,31 @@ async def classify_event_llm(title: str, summary: str = "") -> Dict[str, Any]:
|
|
| 153 |
Resilient Multi-Engine Classification Gateway.
|
| 154 |
Tries Groq -> Gemini -> HF -> Regex Fallback.
|
| 155 |
"""
|
| 156 |
-
prompt = f"""
|
| 157 |
-
|
| 158 |
-
|
| 159 |
-
|
| 160 |
-
|
| 161 |
-
|
| 162 |
-
|
| 163 |
-
|
| 164 |
-
|
| 165 |
-
|
| 166 |
-
|
| 167 |
-
|
| 168 |
-
|
| 169 |
-
|
| 170 |
-
|
| 171 |
-
|
| 172 |
-
|
| 173 |
-
|
| 174 |
-
|
| 175 |
-
|
| 176 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 177 |
|
| 178 |
# ENGINE TIER 1: GROQ
|
| 179 |
res = await classify_with_groq(prompt)
|
|
@@ -203,21 +207,36 @@ async def classify_event_llm(title: str, summary: str = "") -> Dict[str, Any]:
|
|
| 203 |
}
|
| 204 |
|
| 205 |
def parse_llm_res(res: Dict, provider: str) -> Dict[str, Any]:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 206 |
return {
|
|
|
|
| 207 |
"category": res.get("category", "GENERAL").upper(),
|
| 208 |
"severity_score": float(res.get("severity_score", 3.0)),
|
| 209 |
-
"
|
|
|
|
| 210 |
"location_admin1": res.get("location_admin1"),
|
| 211 |
"country": res.get("location_country"),
|
| 212 |
"location_raw": res.get("location_text_span"),
|
| 213 |
-
"extraction_confidence":
|
| 214 |
-
"tags":
|
| 215 |
"event_type": res.get("event_type", "Other"),
|
| 216 |
"actor1": res.get("actor1"),
|
| 217 |
"actor2": res.get("actor2"),
|
|
|
|
| 218 |
"fatalities": int(res.get("fatalities", 0)),
|
| 219 |
-
"notes": res.get(
|
| 220 |
-
"ai_analysis": res.get("deep_analysis", ""),
|
| 221 |
"ai_classified": True,
|
| 222 |
"provider": provider
|
| 223 |
}
|
|
|
|
| 153 |
Resilient Multi-Engine Classification Gateway.
|
| 154 |
Tries Groq -> Gemini -> HF -> Regex Fallback.
|
| 155 |
"""
|
| 156 |
+
prompt = f"""Analyze this news headline for an ELITE conflict intelligence database.
|
| 157 |
+
Your goal is absolute accuracy. Only classify REAL kinetic/military/violent incidents.
|
| 158 |
+
Reject diplomacy, politics, economics, sports, or opinion articles.
|
| 159 |
+
|
| 160 |
+
Title: {title}
|
| 161 |
+
Summary: {summary}
|
| 162 |
+
|
| 163 |
+
Return a JSON object with these fields:
|
| 164 |
+
- is_conflict_tactical: Boolean. True ONLY if this is a real kinetic event (explosions, airstrikes, armed clashes, shootings, bombings, military operations). False for diplomacy, sanctions, protests, elections, humanitarian aid, opinion pieces, or general politics.
|
| 165 |
+
- validation_logic: 1 sentence explaining your reasoning.
|
| 166 |
+
- category: MILITARY, TERRORIST, MILITANT, CIVIL_UNREST, or GENERAL
|
| 167 |
+
- strategic_significance: CRITICAL, HIGH, MEDIUM, or LOW
|
| 168 |
+
- event_type: Armed Clash, Airstrike / Artillery, Terrorist Attack, Drone Strike, Naval Engagement, Arrests / Detainment, or Other
|
| 169 |
+
- location_landmark: Specific facility name if mentioned (base, airport, port, checkpoint). null if none.
|
| 170 |
+
- location_city: Specific city or town. null if unclear.
|
| 171 |
+
- location_admin1: Province, state, or oblast name. null if unclear.
|
| 172 |
+
- location_country: Country name. Required.
|
| 173 |
+
- location_text_span: The exact phrase from the title referring to the location.
|
| 174 |
+
- confidence_score: Float 0.0-1.0 indicating overall data reliability.
|
| 175 |
+
- actor1: Primary armed group or military force. Be specific (e.g. "72nd Mechanized Brigade" not just "Ukraine").
|
| 176 |
+
- actor2: Secondary entity or target. null if unknown.
|
| 177 |
+
- weapon: Specific weapon system if mentioned (e.g. "Shahed-136", "HIMARS", "RPG-7"). null if not mentioned.
|
| 178 |
+
- fatalities: Integer count. 0 if not mentioned or unknown.
|
| 179 |
+
- severity_score: Float 0-10 based on tactical impact and scale.
|
| 180 |
+
"""
|
| 181 |
|
| 182 |
# ENGINE TIER 1: GROQ
|
| 183 |
res = await classify_with_groq(prompt)
|
|
|
|
| 207 |
}
|
| 208 |
|
| 209 |
def parse_llm_res(res: Dict, provider: str) -> Dict[str, Any]:
|
| 210 |
+
# World Monitor Noise Gate: Reject non-conflict or low-confidence events
|
| 211 |
+
is_tactical = res.get("is_conflict_tactical")
|
| 212 |
+
confidence = 0.0
|
| 213 |
+
try:
|
| 214 |
+
confidence = float(res.get("confidence_score", 0))
|
| 215 |
+
except (ValueError, TypeError):
|
| 216 |
+
confidence = 0.0
|
| 217 |
+
|
| 218 |
+
if not is_tactical or confidence < 0.7:
|
| 219 |
+
return {"is_noise": True, "logic": res.get("validation_logic", "Rejected by noise gate")}
|
| 220 |
+
|
| 221 |
+
significance = res.get("strategic_significance", "LOW")
|
| 222 |
+
|
| 223 |
return {
|
| 224 |
+
"is_noise": False,
|
| 225 |
"category": res.get("category", "GENERAL").upper(),
|
| 226 |
"severity_score": float(res.get("severity_score", 3.0)),
|
| 227 |
+
"strategic_relevance": significance,
|
| 228 |
+
"location": res.get("location_landmark") or res.get("location_city"),
|
| 229 |
"location_admin1": res.get("location_admin1"),
|
| 230 |
"country": res.get("location_country"),
|
| 231 |
"location_raw": res.get("location_text_span"),
|
| 232 |
+
"extraction_confidence": confidence,
|
| 233 |
+
"tags": [t for t in [res.get("actor1"), res.get("weapon")] if t],
|
| 234 |
"event_type": res.get("event_type", "Other"),
|
| 235 |
"actor1": res.get("actor1"),
|
| 236 |
"actor2": res.get("actor2"),
|
| 237 |
+
"weapon": res.get("weapon"),
|
| 238 |
"fatalities": int(res.get("fatalities", 0)),
|
| 239 |
+
"notes": f"[{significance}] {res.get('validation_logic', '')}",
|
|
|
|
| 240 |
"ai_classified": True,
|
| 241 |
"provider": provider
|
| 242 |
}
|
poller/db_inserter.py
CHANGED
|
@@ -33,6 +33,11 @@ async def get_pool():
|
|
| 33 |
async def upsert_event(event: Dict[str, Any]):
|
| 34 |
pool = await get_pool()
|
| 35 |
async with pool.acquire() as conn:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 36 |
# Use Conflict Tracker to group events
|
| 37 |
try:
|
| 38 |
conflict_id, conflict_name = await identify_or_create_conflict(conn, event)
|
|
@@ -40,6 +45,33 @@ async def upsert_event(event: Dict[str, Any]):
|
|
| 40 |
log.warning(f"Conflict tracking failed: {e}")
|
| 41 |
conflict_id, conflict_name = None, None
|
| 42 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 43 |
query = """
|
| 44 |
INSERT INTO conflict_events (
|
| 45 |
event_id, source, source_reliability, event_time, event_date,
|
|
@@ -51,13 +83,13 @@ async def upsert_event(event: Dict[str, Any]):
|
|
| 51 |
severity, severity_score, title, notes, tags, source_url, category,
|
| 52 |
conflict_id, conflict_name,
|
| 53 |
geo_confidence, geo_method, geocode_provider, location_raw,
|
| 54 |
-
ai_analysis
|
| 55 |
) VALUES (
|
| 56 |
$1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11,
|
| 57 |
$12::double precision, $13::double precision, ST_SetSRID(ST_MakePoint($13::double precision, $12::double precision), 4326), $14,
|
| 58 |
$15, $16, $17, $18, $19, $20, $21,
|
| 59 |
$22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33,
|
| 60 |
-
$34, $35, $36, $37, $38
|
| 61 |
)
|
| 62 |
ON CONFLICT (event_id) DO UPDATE SET
|
| 63 |
severity_score = EXCLUDED.severity_score,
|
|
@@ -70,6 +102,7 @@ async def upsert_event(event: Dict[str, Any]):
|
|
| 70 |
ai_analysis = EXCLUDED.ai_analysis,
|
| 71 |
conflict_id = EXCLUDED.conflict_id,
|
| 72 |
conflict_name = EXCLUDED.conflict_name,
|
|
|
|
| 73 |
ingested_at = NOW();
|
| 74 |
"""
|
| 75 |
|
|
@@ -111,7 +144,8 @@ async def upsert_event(event: Dict[str, Any]):
|
|
| 111 |
event.get("geo_method"),
|
| 112 |
event.get("geocode_provider"),
|
| 113 |
event.get("location_raw"),
|
| 114 |
-
event.get("ai_analysis")
|
|
|
|
| 115 |
)
|
| 116 |
|
| 117 |
try:
|
|
@@ -123,6 +157,7 @@ async def upsert_event(event: Dict[str, Any]):
|
|
| 123 |
"title": event["title"],
|
| 124 |
"city": event.get("city"),
|
| 125 |
"country": event["country"],
|
|
|
|
| 126 |
"lat": event["lat"],
|
| 127 |
"lon": event["lon"],
|
| 128 |
"geo_precision": event.get("geo_precision", 3),
|
|
@@ -135,7 +170,11 @@ async def upsert_event(event: Dict[str, Any]):
|
|
| 135 |
"weapon": event.get("weapon"),
|
| 136 |
"fatalities": event.get("fatalities", 0),
|
| 137 |
"notes": event.get("notes", ""),
|
| 138 |
-
"ai_analysis": event.get("ai_analysis", "")
|
|
|
|
|
|
|
|
|
|
|
|
|
| 139 |
}
|
| 140 |
await conn.execute(f"SELECT pg_notify('new_conflict_event', $1)", json.dumps(payload))
|
| 141 |
log.debug(f"Upserted and Enriched Notified event {event['event_id']}")
|
|
|
|
| 33 |
async def upsert_event(event: Dict[str, Any]):
|
| 34 |
pool = await get_pool()
|
| 35 |
async with pool.acquire() as conn:
|
| 36 |
+
# Guard: Reject events that failed geocoding entirely
|
| 37 |
+
if event.get("lat", 0) == 0.0 and event.get("lon", 0) == 0.0:
|
| 38 |
+
log.warning(f"Dropping event with (0,0) coordinates: {event.get('title', '')[:60]}")
|
| 39 |
+
return None
|
| 40 |
+
|
| 41 |
# Use Conflict Tracker to group events
|
| 42 |
try:
|
| 43 |
conflict_id, conflict_name = await identify_or_create_conflict(conn, event)
|
|
|
|
| 45 |
log.warning(f"Conflict tracking failed: {e}")
|
| 46 |
conflict_id, conflict_name = None, None
|
| 47 |
|
| 48 |
+
# --- WORLD MONITOR TRIANGULATION: Proximity Search ---
|
| 49 |
+
# Search for an event within 10km and +/- 12 hours
|
| 50 |
+
search_query = """
|
| 51 |
+
SELECT event_id, verification_count, source_urls FROM conflict_events
|
| 52 |
+
WHERE ST_DWithin(geom, ST_SetSRID(ST_MakePoint($1, $2), 4326), 10000)
|
| 53 |
+
AND event_time >= $3::timestamp - INTERVAL '12 hours'
|
| 54 |
+
AND event_time <= $3::timestamp + INTERVAL '12 hours'
|
| 55 |
+
LIMIT 1;
|
| 56 |
+
"""
|
| 57 |
+
existing = await conn.fetchrow(search_query, event["lon"], event["lat"], event["event_time"])
|
| 58 |
+
|
| 59 |
+
if existing:
|
| 60 |
+
# INTERFACE: Update existing event with new source and increment verification
|
| 61 |
+
log.info(f"Triangulation Successful: Merging {event['event_id']} with existing {existing['event_id']}")
|
| 62 |
+
update_query = """
|
| 63 |
+
UPDATE conflict_events SET
|
| 64 |
+
verification_count = verification_count + 1,
|
| 65 |
+
source_urls = array_append(source_urls, $2),
|
| 66 |
+
severity_score = GREATEST(severity_score, $3),
|
| 67 |
+
notes = notes || '\n[UPDATE] ' || $4,
|
| 68 |
+
ingested_at = NOW()
|
| 69 |
+
WHERE event_id = $1;
|
| 70 |
+
"""
|
| 71 |
+
await conn.execute(update_query, existing["event_id"], event.get("source_url"), event.get("severity_score", 0.0), event.get("notes", ""))
|
| 72 |
+
return existing["event_id"]
|
| 73 |
+
|
| 74 |
+
|
| 75 |
query = """
|
| 76 |
INSERT INTO conflict_events (
|
| 77 |
event_id, source, source_reliability, event_time, event_date,
|
|
|
|
| 83 |
severity, severity_score, title, notes, tags, source_url, category,
|
| 84 |
conflict_id, conflict_name,
|
| 85 |
geo_confidence, geo_method, geocode_provider, location_raw,
|
| 86 |
+
ai_analysis, strategic_relevance, source_urls
|
| 87 |
) VALUES (
|
| 88 |
$1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11,
|
| 89 |
$12::double precision, $13::double precision, ST_SetSRID(ST_MakePoint($13::double precision, $12::double precision), 4326), $14,
|
| 90 |
$15, $16, $17, $18, $19, $20, $21,
|
| 91 |
$22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33,
|
| 92 |
+
$34, $35, $36, $37, $38, $39, ARRAY[$30]::text[]
|
| 93 |
)
|
| 94 |
ON CONFLICT (event_id) DO UPDATE SET
|
| 95 |
severity_score = EXCLUDED.severity_score,
|
|
|
|
| 102 |
ai_analysis = EXCLUDED.ai_analysis,
|
| 103 |
conflict_id = EXCLUDED.conflict_id,
|
| 104 |
conflict_name = EXCLUDED.conflict_name,
|
| 105 |
+
strategic_relevance = EXCLUDED.strategic_relevance,
|
| 106 |
ingested_at = NOW();
|
| 107 |
"""
|
| 108 |
|
|
|
|
| 144 |
event.get("geo_method"),
|
| 145 |
event.get("geocode_provider"),
|
| 146 |
event.get("location_raw"),
|
| 147 |
+
event.get("ai_analysis"),
|
| 148 |
+
event.get("strategic_relevance", "LOW")
|
| 149 |
)
|
| 150 |
|
| 151 |
try:
|
|
|
|
| 157 |
"title": event["title"],
|
| 158 |
"city": event.get("city"),
|
| 159 |
"country": event["country"],
|
| 160 |
+
"country_iso3": event.get("country_iso3"),
|
| 161 |
"lat": event["lat"],
|
| 162 |
"lon": event["lon"],
|
| 163 |
"geo_precision": event.get("geo_precision", 3),
|
|
|
|
| 170 |
"weapon": event.get("weapon"),
|
| 171 |
"fatalities": event.get("fatalities", 0),
|
| 172 |
"notes": event.get("notes", ""),
|
| 173 |
+
"ai_analysis": event.get("ai_analysis", ""),
|
| 174 |
+
"strategic_relevance": event.get("strategic_relevance", "LOW"),
|
| 175 |
+
"verification_count": 1,
|
| 176 |
+
"source_urls": [event.get("source_url", "")],
|
| 177 |
+
"source_url": event.get("source_url", "")
|
| 178 |
}
|
| 179 |
await conn.execute(f"SELECT pg_notify('new_conflict_event', $1)", json.dumps(payload))
|
| 180 |
log.debug(f"Upserted and Enriched Notified event {event['event_id']}")
|
poller/deduplicator.py
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import hashlib
|
| 2 |
+
import logging
|
| 3 |
+
from api.database import db
|
| 4 |
+
|
| 5 |
+
log = logging.getLogger(__name__)
|
| 6 |
+
|
| 7 |
+
async def is_duplicate_event(title: str, window_days: int = 1) -> bool:
|
| 8 |
+
"""
|
| 9 |
+
Checks if a similar headline has been processed recently.
|
| 10 |
+
Uses semantic hashing (normalized title) and Redis/Mem storage.
|
| 11 |
+
"""
|
| 12 |
+
if not title: return False
|
| 13 |
+
|
| 14 |
+
# Simple normalization for semantic hashing
|
| 15 |
+
normalized = "".join(filter(str.isalnum, title.lower()))
|
| 16 |
+
hash_key = hashlib.md5(normalized.encode()).hexdigest()
|
| 17 |
+
|
| 18 |
+
redis_key = f"seen_event:{hash_key}"
|
| 19 |
+
|
| 20 |
+
if db.redis:
|
| 21 |
+
exists = await db.redis.get(redis_key)
|
| 22 |
+
if exists:
|
| 23 |
+
log.debug(f"Token Saved: Skipping duplicate event: {title[:50]}...")
|
| 24 |
+
return True
|
| 25 |
+
|
| 26 |
+
# Mark as seen for X days
|
| 27 |
+
await db.redis.setex(redis_key, 86400 * window_days, "1")
|
| 28 |
+
|
| 29 |
+
return False
|
poller/gdelt_poller.py
CHANGED
|
@@ -7,38 +7,46 @@ from datetime import datetime, timezone
|
|
| 7 |
from poller.db_inserter import upsert_event
|
| 8 |
from poller.geo_utils import geocode_ranked
|
| 9 |
from poller.classifier import classify_event_llm
|
|
|
|
| 10 |
|
| 11 |
log = logging.getLogger(__name__)
|
| 12 |
GDELT_URL = 'https://api.gdeltproject.org/api/v2/doc/doc'
|
| 13 |
|
| 14 |
-
#
|
| 15 |
NOISE_PATTERNS = [
|
| 16 |
r"years ago", r"anniversary", r"memorial", r"remembers", r"history of",
|
| 17 |
-
r"how to watch", r"commemorate", r"museum", r"exhibition", r"biography"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 18 |
]
|
| 19 |
|
| 20 |
def passes_quality_filter(article):
|
| 21 |
title = article.get('title', '')
|
| 22 |
-
if len(title) <
|
| 23 |
return False
|
| 24 |
|
| 25 |
-
# Filter out historical retrospectives / noise
|
| 26 |
title_lower = title.lower()
|
| 27 |
for pattern in NOISE_PATTERNS:
|
| 28 |
if re.search(pattern, title_lower):
|
| 29 |
-
log.debug(f"
|
| 30 |
return False
|
| 31 |
|
| 32 |
return True
|
| 33 |
|
| 34 |
async def poll_gdelt():
|
| 35 |
-
# Stagger startup to avoid simultaneous polling pressure
|
| 36 |
await asyncio.sleep(10)
|
| 37 |
|
|
|
|
| 38 |
params = {
|
| 39 |
-
'query': '(
|
| 40 |
'mode': 'artlist',
|
| 41 |
-
'maxrecords':
|
| 42 |
'format': 'json',
|
| 43 |
}
|
| 44 |
|
|
@@ -68,23 +76,43 @@ async def poll_gdelt():
|
|
| 68 |
log.info(f"GDELT returned {len(articles)} articles.")
|
| 69 |
|
| 70 |
count = 0
|
|
|
|
|
|
|
|
|
|
|
|
|
| 71 |
for article in articles:
|
| 72 |
if not passes_quality_filter(article):
|
| 73 |
continue
|
| 74 |
|
| 75 |
title = article.get("title", "").replace("\n", " ").strip()
|
| 76 |
|
| 77 |
-
#
|
|
|
|
|
|
|
|
|
|
|
|
|
| 78 |
try:
|
| 79 |
ai_res = await classify_event_llm(title)
|
| 80 |
|
| 81 |
-
#
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 82 |
geo_res = await geocode_ranked(
|
| 83 |
ai_res.get("location"),
|
| 84 |
ai_res.get("country"),
|
| 85 |
ai_res.get("location_admin1")
|
| 86 |
)
|
| 87 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 88 |
uniq = str(uuid.uuid5(uuid.NAMESPACE_URL, article.get('url', ''))).split('-')[0]
|
| 89 |
event_time = datetime.now(timezone.utc).replace(tzinfo=None)
|
| 90 |
|
|
@@ -115,16 +143,17 @@ async def poll_gdelt():
|
|
| 115 |
"actor2": ai_res.get("actor2"),
|
| 116 |
"fatalities": ai_res.get("fatalities", 0),
|
| 117 |
"notes": ai_res.get("notes"),
|
|
|
|
|
|
|
| 118 |
}
|
| 119 |
|
| 120 |
await upsert_event(event)
|
| 121 |
count += 1
|
| 122 |
|
| 123 |
-
# 3. Rate Limit Compliance (2s sleep)
|
| 124 |
await asyncio.sleep(2.0)
|
| 125 |
|
| 126 |
except Exception as e:
|
| 127 |
log.error(f"Error processing GDELT article: {e}")
|
| 128 |
await asyncio.sleep(5)
|
| 129 |
|
| 130 |
-
log.info(f"
|
|
|
|
| 7 |
from poller.db_inserter import upsert_event
|
| 8 |
from poller.geo_utils import geocode_ranked
|
| 9 |
from poller.classifier import classify_event_llm
|
| 10 |
+
from poller.deduplicator import is_duplicate_event
|
| 11 |
|
| 12 |
log = logging.getLogger(__name__)
|
| 13 |
GDELT_URL = 'https://api.gdeltproject.org/api/v2/doc/doc'
|
| 14 |
|
| 15 |
+
# Expanded noise patterns — historical, editorial, and non-kinetic content
|
| 16 |
NOISE_PATTERNS = [
|
| 17 |
r"years ago", r"anniversary", r"memorial", r"remembers", r"history of",
|
| 18 |
+
r"how to watch", r"commemorate", r"museum", r"exhibition", r"biography",
|
| 19 |
+
r"book review", r"opinion:", r"editorial:", r"analysis:", r"podcast",
|
| 20 |
+
r"interview with", r"what you need to know", r"explained:", r"timeline of",
|
| 21 |
+
r"trade war", r"war on drugs", r"war on poverty", r"culture war",
|
| 22 |
+
r"star wars", r"price war", r"bidding war", r"turf war",
|
| 23 |
+
r"fantasy football", r"world cup", r"olympic", r"nba ", r"nfl ",
|
| 24 |
+
r"box office", r"movie", r"tv show", r"streaming", r"netflix",
|
| 25 |
+
r"stock market", r"wall street", r"cryptocurrency", r"bitcoin",
|
| 26 |
+
r"election results", r"campaign trail", r"poll shows", r"voter",
|
| 27 |
]
|
| 28 |
|
| 29 |
def passes_quality_filter(article):
|
| 30 |
title = article.get('title', '')
|
| 31 |
+
if len(title) < 20:
|
| 32 |
return False
|
| 33 |
|
|
|
|
| 34 |
title_lower = title.lower()
|
| 35 |
for pattern in NOISE_PATTERNS:
|
| 36 |
if re.search(pattern, title_lower):
|
| 37 |
+
log.debug(f"GDELT noise filter: {title[:60]}")
|
| 38 |
return False
|
| 39 |
|
| 40 |
return True
|
| 41 |
|
| 42 |
async def poll_gdelt():
|
|
|
|
| 43 |
await asyncio.sleep(10)
|
| 44 |
|
| 45 |
+
# Precision query — only actual kinetic/military terms, not broad "war" or "conflict"
|
| 46 |
params = {
|
| 47 |
+
'query': '(airstrike OR "armed clash" OR shelling OR artillery OR "drone strike" OR missile OR bombing OR "suicide attack" OR ambush OR mortar OR "killed in" OR "troops deployed" OR "military operation") sourcelang:English',
|
| 48 |
'mode': 'artlist',
|
| 49 |
+
'maxrecords': 75,
|
| 50 |
'format': 'json',
|
| 51 |
}
|
| 52 |
|
|
|
|
| 76 |
log.info(f"GDELT returned {len(articles)} articles.")
|
| 77 |
|
| 78 |
count = 0
|
| 79 |
+
skipped_noise = 0
|
| 80 |
+
skipped_dupe = 0
|
| 81 |
+
skipped_geo = 0
|
| 82 |
+
|
| 83 |
for article in articles:
|
| 84 |
if not passes_quality_filter(article):
|
| 85 |
continue
|
| 86 |
|
| 87 |
title = article.get("title", "").replace("\n", " ").strip()
|
| 88 |
|
| 89 |
+
# Token-saving deduplication
|
| 90 |
+
if await is_duplicate_event(title):
|
| 91 |
+
skipped_dupe += 1
|
| 92 |
+
continue
|
| 93 |
+
|
| 94 |
try:
|
| 95 |
ai_res = await classify_event_llm(title)
|
| 96 |
|
| 97 |
+
# World Monitor Noise Gate
|
| 98 |
+
if ai_res.get("is_noise"):
|
| 99 |
+
log.debug(f"GDELT Skeptic Rejection: {title[:60]}... Reason: {ai_res.get('logic')}")
|
| 100 |
+
skipped_noise += 1
|
| 101 |
+
continue
|
| 102 |
+
|
| 103 |
+
# High-Fidelity Geocoding
|
| 104 |
geo_res = await geocode_ranked(
|
| 105 |
ai_res.get("location"),
|
| 106 |
ai_res.get("country"),
|
| 107 |
ai_res.get("location_admin1")
|
| 108 |
)
|
| 109 |
|
| 110 |
+
# Drop failed geocodes — don't pollute the map with (0,0)
|
| 111 |
+
if geo_res.get("confidence", 0) == 0.0 and geo_res.get("method") == "failed":
|
| 112 |
+
log.warning(f"GDELT: Dropping event with failed geocode: {title[:60]}")
|
| 113 |
+
skipped_geo += 1
|
| 114 |
+
continue
|
| 115 |
+
|
| 116 |
uniq = str(uuid.uuid5(uuid.NAMESPACE_URL, article.get('url', ''))).split('-')[0]
|
| 117 |
event_time = datetime.now(timezone.utc).replace(tzinfo=None)
|
| 118 |
|
|
|
|
| 143 |
"actor2": ai_res.get("actor2"),
|
| 144 |
"fatalities": ai_res.get("fatalities", 0),
|
| 145 |
"notes": ai_res.get("notes"),
|
| 146 |
+
"ai_analysis": ai_res.get("ai_analysis"),
|
| 147 |
+
"strategic_relevance": ai_res.get("strategic_relevance", "LOW"),
|
| 148 |
}
|
| 149 |
|
| 150 |
await upsert_event(event)
|
| 151 |
count += 1
|
| 152 |
|
|
|
|
| 153 |
await asyncio.sleep(2.0)
|
| 154 |
|
| 155 |
except Exception as e:
|
| 156 |
log.error(f"Error processing GDELT article: {e}")
|
| 157 |
await asyncio.sleep(5)
|
| 158 |
|
| 159 |
+
log.info(f"GDELT: Processed {count} events. Rejected {skipped_noise} noise, {skipped_dupe} dupes, {skipped_geo} bad geo.")
|
poller/geo_utils.py
CHANGED
|
@@ -10,24 +10,68 @@ import json
|
|
| 10 |
log = logging.getLogger(__name__)
|
| 11 |
|
| 12 |
# Tactical Centroids for major conflict zones (High precision manual overrides)
|
| 13 |
-
#
|
| 14 |
HOTSPOTS = {
|
|
|
|
| 15 |
"gaza": (31.3547, 34.3088, "Palestine", "PSE"),
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 16 |
"west bank": (31.9522, 35.2332, "Palestine", "PSE"),
|
|
|
|
|
|
|
|
|
|
| 17 |
"donbas": (48.0, 37.8, "Ukraine", "UKR"),
|
| 18 |
"donetsk": (48.0196, 37.8018, "Ukraine", "UKR"),
|
| 19 |
"luhansk": (48.574, 39.307, "Ukraine", "UKR"),
|
| 20 |
"kharkiv": (49.9935, 36.2304, "Ukraine", "UKR"),
|
| 21 |
"avdiivka": (48.1366, 37.7347, "Ukraine", "UKR"),
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 22 |
"red sea": (20.0, 38.0, "International Waters", "INT"),
|
| 23 |
-
"
|
| 24 |
-
"
|
|
|
|
|
|
|
| 25 |
"darfur": (13.0, 25.0, "Sudan", "SDN"),
|
| 26 |
"khartoum": (15.5007, 32.5599, "Sudan", "SDN"),
|
|
|
|
|
|
|
|
|
|
| 27 |
"tigray": (13.5, 39.0, "Ethiopia", "ETH"),
|
|
|
|
|
|
|
| 28 |
"kabul": (34.555, 69.207, "Afghanistan", "AFG"),
|
|
|
|
|
|
|
|
|
|
| 29 |
"mali": (17.57, -3.99, "Mali", "MLI"),
|
|
|
|
|
|
|
| 30 |
"hormuz": (26.5, 56.5, "International Waters", "INT"),
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 31 |
}
|
| 32 |
|
| 33 |
COUNTRY_CENTROIDS = {
|
|
@@ -64,8 +108,9 @@ def get_geolocator():
|
|
| 64 |
def normalize_location_text(text: str) -> str:
|
| 65 |
if not text: return ""
|
| 66 |
t = text.lower()
|
| 67 |
-
# Remove noise patterns
|
| 68 |
t = re.sub(r'\b(near|outside|north of|south of|east of|west of|frontline near|region of|area of)\b', '', t)
|
|
|
|
| 69 |
t = re.sub(r'\b(city|province|district|oblast|state)\b', '', t)
|
| 70 |
# Simplify whitespace
|
| 71 |
return ' '.join(t.split()).strip()
|
|
@@ -126,9 +171,14 @@ async def geocode_ranked(place: str, country: str, admin1: str = None) -> Dict:
|
|
| 126 |
if admin1 and (admin1.lower() in str(addr).lower()):
|
| 127 |
score += 0.20
|
| 128 |
|
| 129 |
-
#
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 130 |
if clean_place in cand.address.lower():
|
| 131 |
-
score += 0.
|
| 132 |
|
| 133 |
# Feature Type Preference (10%)
|
| 134 |
if addr.get('city') or addr.get('town') or addr.get('village'):
|
|
|
|
| 10 |
log = logging.getLogger(__name__)
|
| 11 |
|
| 12 |
# Tactical Centroids for major conflict zones (High precision manual overrides)
|
| 13 |
+
# These bypass Nominatim entirely for reliable, instant geocoding of known hotspots.
|
| 14 |
HOTSPOTS = {
|
| 15 |
+
# --- UKRAINE FRONTLINE ---
|
| 16 |
"gaza": (31.3547, 34.3088, "Palestine", "PSE"),
|
| 17 |
+
"gaza strip": (31.3547, 34.3088, "Palestine", "PSE"),
|
| 18 |
+
"rafah": (31.2768, 34.2457, "Palestine", "PSE"),
|
| 19 |
+
"khan younis": (31.3462, 34.3015, "Palestine", "PSE"),
|
| 20 |
+
"khan yunis": (31.3462, 34.3015, "Palestine", "PSE"),
|
| 21 |
+
"jabalia": (31.5281, 34.4831, "Palestine", "PSE"),
|
| 22 |
+
"deir al-balah": (31.4180, 34.3509, "Palestine", "PSE"),
|
| 23 |
+
"nuseirat": (31.4416, 34.3963, "Palestine", "PSE"),
|
| 24 |
"west bank": (31.9522, 35.2332, "Palestine", "PSE"),
|
| 25 |
+
"jenin": (32.4610, 35.3008, "Palestine", "PSE"),
|
| 26 |
+
"tulkarm": (32.3104, 35.0286, "Palestine", "PSE"),
|
| 27 |
+
# --- UKRAINE ---
|
| 28 |
"donbas": (48.0, 37.8, "Ukraine", "UKR"),
|
| 29 |
"donetsk": (48.0196, 37.8018, "Ukraine", "UKR"),
|
| 30 |
"luhansk": (48.574, 39.307, "Ukraine", "UKR"),
|
| 31 |
"kharkiv": (49.9935, 36.2304, "Ukraine", "UKR"),
|
| 32 |
"avdiivka": (48.1366, 37.7347, "Ukraine", "UKR"),
|
| 33 |
+
"bakhmut": (48.5954, 38.0003, "Ukraine", "UKR"),
|
| 34 |
+
"zaporizhzhia": (47.8388, 35.1396, "Ukraine", "UKR"),
|
| 35 |
+
"kherson": (46.6354, 32.6169, "Ukraine", "UKR"),
|
| 36 |
+
"mariupol": (47.0958, 37.5494, "Ukraine", "UKR"),
|
| 37 |
+
"pokrovsk": (48.2833, 37.1833, "Ukraine", "UKR"),
|
| 38 |
+
"kursk": (51.7304, 36.1927, "Russia", "RUS"),
|
| 39 |
+
"crimea": (45.3, 34.1, "Ukraine", "UKR"),
|
| 40 |
+
"odesa": (46.4825, 30.7233, "Ukraine", "UKR"),
|
| 41 |
+
"kyiv": (50.4501, 30.5234, "Ukraine", "UKR"),
|
| 42 |
+
# --- RED SEA / YEMEN ---
|
| 43 |
"red sea": (20.0, 38.0, "International Waters", "INT"),
|
| 44 |
+
"hodeidah": (14.7980, 42.9540, "Yemen", "YEM"),
|
| 45 |
+
"sanaa": (15.3694, 44.1910, "Yemen", "YEM"),
|
| 46 |
+
"aden": (12.7855, 45.0187, "Yemen", "YEM"),
|
| 47 |
+
# --- SUDAN ---
|
| 48 |
"darfur": (13.0, 25.0, "Sudan", "SDN"),
|
| 49 |
"khartoum": (15.5007, 32.5599, "Sudan", "SDN"),
|
| 50 |
+
"el fasher": (13.6300, 25.3500, "Sudan", "SDN"),
|
| 51 |
+
"port sudan": (19.6158, 37.2164, "Sudan", "SDN"),
|
| 52 |
+
# --- HORN OF AFRICA ---
|
| 53 |
"tigray": (13.5, 39.0, "Ethiopia", "ETH"),
|
| 54 |
+
"mogadishu": (2.0469, 45.3182, "Somalia", "SOM"),
|
| 55 |
+
# --- ASIA ---
|
| 56 |
"kabul": (34.555, 69.207, "Afghanistan", "AFG"),
|
| 57 |
+
"kandahar": (31.6133, 65.7101, "Afghanistan", "AFG"),
|
| 58 |
+
"myanmar": (21.9162, 95.956, "Myanmar", "MMR"),
|
| 59 |
+
# --- SAHEL ---
|
| 60 |
"mali": (17.57, -3.99, "Mali", "MLI"),
|
| 61 |
+
"burkina faso": (12.3714, -1.5197, "Burkina Faso", "BFA"),
|
| 62 |
+
# --- MARITIME ---
|
| 63 |
"hormuz": (26.5, 56.5, "International Waters", "INT"),
|
| 64 |
+
"taiwan strait": (24.0, 119.0, "Taiwan", "TWN"),
|
| 65 |
+
"south china sea": (12.0, 113.0, "International Waters", "INT"),
|
| 66 |
+
# --- MIDDLE EAST ---
|
| 67 |
+
"aleppo": (36.2021, 37.1343, "Syria", "SYR"),
|
| 68 |
+
"idlib": (35.9306, 36.6339, "Syria", "SYR"),
|
| 69 |
+
"beirut": (33.8938, 35.5018, "Lebanon", "LBN"),
|
| 70 |
+
"baghdad": (33.3152, 44.3661, "Iraq", "IRQ"),
|
| 71 |
+
"mosul": (36.3350, 43.1189, "Iraq", "IRQ"),
|
| 72 |
+
# --- AFRICA ---
|
| 73 |
+
"goma": (-1.6785, 29.2284, "DR Congo", "COD"),
|
| 74 |
+
"beni": (0.4900, 29.4700, "DR Congo", "COD"),
|
| 75 |
}
|
| 76 |
|
| 77 |
COUNTRY_CENTROIDS = {
|
|
|
|
| 108 |
def normalize_location_text(text: str) -> str:
|
| 109 |
if not text: return ""
|
| 110 |
t = text.lower()
|
| 111 |
+
# Remove noise patterns but preserve critical infrastructure identifiers
|
| 112 |
t = re.sub(r'\b(near|outside|north of|south of|east of|west of|frontline near|region of|area of)\b', '', t)
|
| 113 |
+
# Be careful not to remove 'base' or 'port' here
|
| 114 |
t = re.sub(r'\b(city|province|district|oblast|state)\b', '', t)
|
| 115 |
# Simplify whitespace
|
| 116 |
return ' '.join(t.split()).strip()
|
|
|
|
| 171 |
if admin1 and (admin1.lower() in str(addr).lower()):
|
| 172 |
score += 0.20
|
| 173 |
|
| 174 |
+
# PINPOINT LANDMARK BOOST (World Monitor Requirement)
|
| 175 |
+
landmark_kws = ['base', 'airport', 'port', 'checkpoint', 'refinery', 'plant', 'depot', 'harbor', 'dam']
|
| 176 |
+
if any(kw in clean_place for kw in landmark_kws) and any(kw in cand.address.lower() for kw in landmark_kws):
|
| 177 |
+
score += 0.25 # High boost for strategic infrastructure
|
| 178 |
+
|
| 179 |
+
# Name Similarity (Simple check - 15%)
|
| 180 |
if clean_place in cand.address.lower():
|
| 181 |
+
score += 0.15
|
| 182 |
|
| 183 |
# Feature Type Preference (10%)
|
| 184 |
if addr.get('city') or addr.get('town') or addr.get('village'):
|
poller/rss_poller.py
CHANGED
|
@@ -6,6 +6,7 @@ from datetime import datetime, timezone
|
|
| 6 |
from poller.db_inserter import upsert_event
|
| 7 |
from poller.geo_utils import geocode_ranked
|
| 8 |
from poller.classifier import classify_event_llm
|
|
|
|
| 9 |
|
| 10 |
log = logging.getLogger(__name__)
|
| 11 |
|
|
@@ -19,73 +20,97 @@ RSS_FEEDS = [
|
|
| 19 |
]
|
| 20 |
|
| 21 |
CONFLICT_KEYWORDS = ['battle','strike','attack','shelling','airstrike',
|
| 22 |
-
'missile','killed','fatalities','troops','offensive', 'war', 'army'
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
event_time = datetime.now(timezone.utc).replace(tzinfo=None)
|
| 26 |
-
uniq = str(uuid.uuid5(uuid.NAMESPACE_URL, entry.get('link', ''))).split('-')[0]
|
| 27 |
-
return {
|
| 28 |
-
"event_id": f"CIQ-{event_time.strftime('%Y%m%d')}-RSS-{uniq}",
|
| 29 |
-
"source": source,
|
| 30 |
-
"source_reliability": "MEDIUM",
|
| 31 |
-
"event_time": event_time,
|
| 32 |
-
"event_date": event_time.date(),
|
| 33 |
-
"country": geo_res["country"],
|
| 34 |
-
"country_iso3": geo_res["iso3"],
|
| 35 |
-
"admin1": geo_res.get("admin1"),
|
| 36 |
-
"lat": geo_res["lat"],
|
| 37 |
-
"lon": geo_res["lon"],
|
| 38 |
-
"geo_precision": geo_res["precision"],
|
| 39 |
-
"geo_confidence": geo_res["confidence"],
|
| 40 |
-
"geo_method": geo_res["method"],
|
| 41 |
-
"geocode_provider": geo_res["provider"],
|
| 42 |
-
"location_raw": ai_res.get("location_raw"),
|
| 43 |
-
"event_type": ai_res.get("event_type", "Other"),
|
| 44 |
-
"severity": "HIGH" if ai_res["severity_score"] > 7 else "MEDIUM" if ai_res["severity_score"] > 4 else "LOW",
|
| 45 |
-
"severity_score": ai_res.get("severity_score", 3.0),
|
| 46 |
-
"category": ai_res.get("category", "GENERAL"),
|
| 47 |
-
"tags": ai_res.get("tags", []),
|
| 48 |
-
"title": entry.get("title", "")[:500],
|
| 49 |
-
"notes": ai_res.get("notes") or entry.get("summary", "")[:1000],
|
| 50 |
-
"source_url": entry.get("link", ""),
|
| 51 |
-
"actor1": ai_res.get("actor1"),
|
| 52 |
-
"actor2": ai_res.get("actor2"),
|
| 53 |
-
"fatalities": ai_res.get("fatalities", 0)
|
| 54 |
-
}
|
| 55 |
|
| 56 |
async def poll_rss():
|
| 57 |
log.info("Polling RSS Feeds with AI Intelligence...")
|
| 58 |
count = 0
|
|
|
|
|
|
|
|
|
|
| 59 |
for url in RSS_FEEDS:
|
| 60 |
try:
|
| 61 |
feed = feedparser.parse(url)
|
| 62 |
for entry in feed.entries:
|
| 63 |
-
title = entry.get('title','')
|
| 64 |
-
summary = entry.get('summary', '')
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 65 |
|
| 66 |
-
#
|
| 67 |
-
if
|
|
|
|
| 68 |
continue
|
| 69 |
|
| 70 |
-
#
|
| 71 |
-
ai_res = await classify_event_llm(
|
| 72 |
|
| 73 |
-
#
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 74 |
geo_res = await geocode_ranked(
|
| 75 |
ai_res.get("location"),
|
| 76 |
ai_res.get("country"),
|
| 77 |
ai_res.get("location_admin1")
|
| 78 |
)
|
| 79 |
|
| 80 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 81 |
await upsert_event(event)
|
| 82 |
count += 1
|
| 83 |
|
| 84 |
-
#
|
| 85 |
await asyncio.sleep(2.0)
|
| 86 |
|
| 87 |
except Exception as e:
|
| 88 |
log.error(f"Error parsing RSS {url}: {e}")
|
| 89 |
-
await asyncio.sleep(5)
|
| 90 |
|
| 91 |
-
log.info(f"
|
|
|
|
| 6 |
from poller.db_inserter import upsert_event
|
| 7 |
from poller.geo_utils import geocode_ranked
|
| 8 |
from poller.classifier import classify_event_llm
|
| 9 |
+
from poller.deduplicator import is_duplicate_event
|
| 10 |
|
| 11 |
log = logging.getLogger(__name__)
|
| 12 |
|
|
|
|
| 20 |
]
|
| 21 |
|
| 22 |
CONFLICT_KEYWORDS = ['battle','strike','attack','shelling','airstrike',
|
| 23 |
+
'missile','killed','fatalities','troops','offensive', 'war', 'army',
|
| 24 |
+
'bombing', 'explosion', 'drone', 'artillery', 'clashes', 'ambush',
|
| 25 |
+
'insurgent', 'militant', 'terrorist', 'combat', 'mortar']
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 26 |
|
| 27 |
async def poll_rss():
|
| 28 |
log.info("Polling RSS Feeds with AI Intelligence...")
|
| 29 |
count = 0
|
| 30 |
+
skipped_noise = 0
|
| 31 |
+
skipped_dupe = 0
|
| 32 |
+
|
| 33 |
for url in RSS_FEEDS:
|
| 34 |
try:
|
| 35 |
feed = feedparser.parse(url)
|
| 36 |
for entry in feed.entries:
|
| 37 |
+
title = entry.get('title','')
|
| 38 |
+
summary = entry.get('summary', '')
|
| 39 |
+
title_lower = title.lower()
|
| 40 |
+
summary_lower = summary.lower()
|
| 41 |
+
|
| 42 |
+
# Pre-filter: Only process likely conflict news
|
| 43 |
+
if not any(kw in title_lower or kw in summary_lower for kw in CONFLICT_KEYWORDS):
|
| 44 |
+
continue
|
| 45 |
|
| 46 |
+
# Token-saving deduplication
|
| 47 |
+
if await is_duplicate_event(title):
|
| 48 |
+
skipped_dupe += 1
|
| 49 |
continue
|
| 50 |
|
| 51 |
+
# AI-Powered Classification
|
| 52 |
+
ai_res = await classify_event_llm(title, summary)
|
| 53 |
|
| 54 |
+
# World Monitor Noise Gate
|
| 55 |
+
if ai_res.get("is_noise"):
|
| 56 |
+
log.debug(f"RSS Skeptic Rejection: {title[:60]}... Reason: {ai_res.get('logic')}")
|
| 57 |
+
skipped_noise += 1
|
| 58 |
+
continue
|
| 59 |
+
|
| 60 |
+
# High-Fidelity Geocoding
|
| 61 |
geo_res = await geocode_ranked(
|
| 62 |
ai_res.get("location"),
|
| 63 |
ai_res.get("country"),
|
| 64 |
ai_res.get("location_admin1")
|
| 65 |
)
|
| 66 |
|
| 67 |
+
# Drop failed geocodes — don't pollute the map
|
| 68 |
+
if geo_res.get("confidence", 0) == 0.0 and geo_res.get("method") == "failed":
|
| 69 |
+
log.warning(f"RSS: Dropping event with failed geocode: {title[:60]}")
|
| 70 |
+
continue
|
| 71 |
+
|
| 72 |
+
event_time = datetime.now(timezone.utc).replace(tzinfo=None)
|
| 73 |
+
uniq = str(uuid.uuid5(uuid.NAMESPACE_URL, entry.get('link', ''))).split('-')[0]
|
| 74 |
+
|
| 75 |
+
event = {
|
| 76 |
+
"event_id": f"CIQ-{event_time.strftime('%Y%m%d')}-RSS-{uniq}",
|
| 77 |
+
"source": "RSS",
|
| 78 |
+
"source_reliability": "MEDIUM",
|
| 79 |
+
"event_time": event_time,
|
| 80 |
+
"event_date": event_time.date(),
|
| 81 |
+
"country": geo_res["country"],
|
| 82 |
+
"country_iso3": geo_res["iso3"],
|
| 83 |
+
"admin1": geo_res.get("admin1"),
|
| 84 |
+
"lat": geo_res["lat"],
|
| 85 |
+
"lon": geo_res["lon"],
|
| 86 |
+
"geo_precision": geo_res["precision"],
|
| 87 |
+
"geo_confidence": geo_res["confidence"],
|
| 88 |
+
"geo_method": geo_res["method"],
|
| 89 |
+
"geocode_provider": geo_res["provider"],
|
| 90 |
+
"location_raw": ai_res.get("location_raw"),
|
| 91 |
+
"event_type": ai_res.get("event_type", "Other"),
|
| 92 |
+
"severity": "HIGH" if ai_res.get("severity_score", 0) > 7 else "MEDIUM" if ai_res.get("severity_score", 0) > 4 else "LOW",
|
| 93 |
+
"severity_score": ai_res.get("severity_score", 3.0),
|
| 94 |
+
"category": ai_res.get("category", "GENERAL"),
|
| 95 |
+
"tags": ai_res.get("tags", []),
|
| 96 |
+
"title": title[:500],
|
| 97 |
+
"notes": ai_res.get("notes"),
|
| 98 |
+
"source_url": entry.get("link", ""),
|
| 99 |
+
"actor1": ai_res.get("actor1"),
|
| 100 |
+
"actor2": ai_res.get("actor2"),
|
| 101 |
+
"fatalities": ai_res.get("fatalities", 0),
|
| 102 |
+
"ai_analysis": ai_res.get("ai_analysis"),
|
| 103 |
+
"strategic_relevance": ai_res.get("strategic_relevance", "LOW"),
|
| 104 |
+
}
|
| 105 |
+
|
| 106 |
await upsert_event(event)
|
| 107 |
count += 1
|
| 108 |
|
| 109 |
+
# Rate limit compliance
|
| 110 |
await asyncio.sleep(2.0)
|
| 111 |
|
| 112 |
except Exception as e:
|
| 113 |
log.error(f"Error parsing RSS {url}: {e}")
|
| 114 |
+
await asyncio.sleep(5)
|
| 115 |
|
| 116 |
+
log.info(f"RSS: Processed {count} events. Rejected {skipped_noise} noise, {skipped_dupe} duplicates.")
|