In [ ]:
import ee
import os
from google.colab import drive
# 1. SETUP
drive.mount('/content/drive', force_remount=True)
try:
ee.Initialize(project='[REDACTED_FOR_SECURITY]')
except:
ee.Authenticate()
ee.Initialize(project='[REDACTED_FOR_SECURITY]')
# CONFIG
WHEAT_MASK_ASSET = '[REDACTED_FOR_SECURITY]'
START_DATE = '2023-10-01'
END_DATE = '2024-04-30'
OUTPUT_FOLDER = 'PhD/Obj1'
OUTPUT_FILENAME = 'Wheat_Test_50_50_Pivot'
# --- 1. POINT SAMPLING (50 WHEAT / 50 NON-WHEAT) ---
wheat_mask = ee.Image(WHEAT_MASK_ASSET)
bounds = wheat_mask.geometry()
# Get 50 Wheat (Class 1)
wheat_pts = ee.FeatureCollection.randomPoints(region=bounds, points=500, seed=42) \
.map(lambda f: f.set('class', wheat_mask.reduceRegion(ee.Reducer.first(), f.geometry(), 10).get('b1'))) \
.filter(ee.Filter.eq('class', 1)).limit(50)
# Get 50 Non-Wheat (Class 0)
non_wheat_pts = ee.FeatureCollection.randomPoints(region=bounds, points=500, seed=99) \
.map(lambda f: f.set('class', wheat_mask.reduceRegion(ee.Reducer.first(), f.geometry(), 10).get('b1'))) \
.filter(ee.Filter.eq('class', 0)).limit(50)
points_for_analysis = wheat_pts.merge(non_wheat_pts)
print(f"Points Generated: {points_for_analysis.size().getInfo()}")
def apply_lee_filter(image):
def lee_single(b):
img_band = image.select(b)
mean = img_band.reduceNeighborhood(ee.Reducer.mean(), ee.Kernel.square(3))
variance = img_band.reduceNeighborhood(ee.Reducer.variance(), ee.Kernel.square(3))
overall_var_img = ee.Image.constant(0.004)
k = variance.divide(variance.add(overall_var_img))
return mean.add(k.multiply(img_band.subtract(mean))).rename(b)
return image.addBands(lee_single('VV'), overwrite=True).addBands(lee_single('VH'), overwrite=True)
def add_ndvi(image):
ndvi = image.normalizedDifference(['B8', 'B4']).rename('NDVI')
return image.addBands(ndvi)
def maskS2clouds(image):
qa = image.select('QA60')
mask = qa.bitwiseAnd(1<<10).eq(0).And(qa.bitwiseAnd(1<<11).eq(0))
return image.updateMask(mask).select(['NDVI']).copyProperties(image, ["system:time_start"])
# S2
s2_col = ee.ImageCollection('COPERNICUS/S2_SR_HARMONIZED') \
.filterDate(START_DATE, END_DATE) \
.filterBounds(bounds) \
.filter(ee.Filter.lt('CLOUDY_PIXEL_PERCENTAGE', 20)) \
.map(add_ndvi) \
.map(maskS2clouds) \
.select('NDVI')
# S1
s1_col = ee.ImageCollection('COPERNICUS/S1_GRD') \
.filterDate(START_DATE, END_DATE) \
.filterBounds(bounds) \
.filter(ee.Filter.eq('instrumentMode', 'IW')) \
.filter(ee.Filter.eq('orbitProperties_pass', 'DESCENDING')) \
.map(apply_lee_filter)
# Daily Logic
def make_daily(col, band):
dates = col.aggregate_array('system:time_start').map(lambda t: ee.Date(t).format('YYYY-MM-dd')).distinct()
def get_img(d_str):
d = ee.Date(d_str)
return col.filterDate(d, d.advance(1, 'day')).median().set('system:time_start', d.millis())
return ee.ImageCollection.fromImages(dates.map(get_img)).filter(ee.Filter.listContains('system:band_names', band))
daily_ndvi_collection = make_daily(s2_col, 'NDVI')
daily_vv_collection = make_daily(s1_col.select('VV'), 'VV')
daily_vh_collection = make_daily(s1_col.select('VH'), 'VH')
def extract_band_time_series(point, collection, band_name, prefix):
# This is the function you pasted
def get_value(image):
value = image.reduceRegion(
reducer=ee.Reducer.mean(),
geometry=point.geometry(),
scale=10
).get(band_name)
# Default to 0 if no value
value = ee.Algorithms.If(value, value, ee.Number(0))
# Prefix key
date_key = ee.String(prefix).cat('_').cat(image.date().format('YYYY-MM-dd'))
return ee.Feature(None, {'value': value, 'date_key': date_key})
# Map over collection
fc_point = collection.select(band_name).map(get_value)
# Filter Nulls
fc_point_filtered = fc_point.filter(ee.Filter.notNull(['value']))
# Server-side Lists
date_keys = fc_point_filtered.aggregate_array('date_key')
values = fc_point_filtered.aggregate_array('value')
return ee.Dictionary.fromLists(date_keys, values)
def get_all_time_series_for_point(point):
# Extract Dicts
ndvi_dict = extract_band_time_series(point, daily_ndvi_collection, 'NDVI', 'NDVI')
vv_dict = extract_band_time_series(point, daily_vv_collection, 'VV', 'VV')
vh_dict = extract_band_time_series(point, daily_vh_collection, 'VH', 'VH')
# Combine
combined_dict = ndvi_dict.combine(vv_dict).combine(vh_dict)
# Add Lat/Lon
coords = point.geometry().coordinates()
return ee.Feature(None).set({
'lon': coords.get(0),
'lat': coords.get(1),
'class': point.get('class'),
'system:index': point.get('system:index')
}).set(combined_dict)
# Map over points
pivoted_time_series = points_for_analysis.map(get_all_time_series_for_point)
# Drop system stuff for export
def remove_system_properties(feature):
properties = feature.propertyNames()
system_property_filter = ee.Filter.stringStartsWith('item', 'system:')
properties_to_keep = properties.filter(system_property_filter.Not())
return feature.select(properties_to_keep)
final_collection = pivoted_time_series.map(remove_system_properties)
task = ee.batch.Export.table.toDrive(
collection=final_collection,
description=OUTPUT_FILENAME,
folder=OUTPUT_FOLDER,
fileNamePrefix=OUTPUT_FILENAME,
fileFormat='CSV'
)
task.start()
print("Export Started. You will get ONE CSV file.")
Mounted at /content/drive Points Generated: 100 Export Started. You will get ONE CSV file.
In [ ]:
import pandas as pd
import numpy as np
import io
# CONFIGURATION
INPUT_FILE = '/content/drive/MyDrive/PhD Obj1/Wheat_Test_50_50_Pivot.csv'
OUTPUT_FILE = '/content/drive/MyDrive/PhD Obj1/Wheat_Test_50_50_Aggregated.csv'
try:
df = pd.read_csv(INPUT_FILE)
print(f"Successfully loaded '{INPUT_FILE}'.")
print("--- Treating 0 values as 'No Data' (NaN) ONLY for Sensor Columns ---")
# Apply NaN replacement strictly to sensor columns
sensor_cols = [c for c in df.columns if c.startswith(('NDVI', 'VV', 'VH'))]
df[sensor_cols] = df[sensor_cols].replace(0, np.nan)
print(f"Original data has {df.shape[0]} rows and {df.shape[1]} columns.")
# 2. LINEAR CONVERSION
print("\nConverting VV and VH from dB to linear scale...")
# Identify VV and VH columns
vv_cols_to_convert = [col for col in df.columns if col.startswith('VV_')]
vh_cols_to_convert = [col for col in df.columns if col.startswith('VH_')]
if vv_cols_to_convert:
df[vv_cols_to_convert] = 10**(df[vv_cols_to_convert] / 10)
print(f"Converted {len(vv_cols_to_convert)} VV columns to linear scale.")
if vh_cols_to_convert:
df[vh_cols_to_convert] = 10**(df[vh_cols_to_convert] / 10)
print(f"Converted {len(vh_cols_to_convert)} VH columns to linear scale.")
# 3. IDENTIFY METRIC COLUMNS
ndvi_cols_raw = [col for col in df.columns if col.startswith('NDVI_')]
vv_cols_raw = [col for col in df.columns if col.startswith('VV_')]
vh_cols_raw = [col for col in df.columns if col.startswith('VH_')]
# 4. FILTERING (>80% Missing Values)
print("\nStep 1: Filtering samples with more than 80% missing values...")
# Calculate missing percentage for each metric
miss_ndvi = df[ndvi_cols_raw].isnull().sum(axis=1) / len(ndvi_cols_raw) if ndvi_cols_raw else pd.Series(0, index=df.index)
miss_vv = df[vv_cols_raw].isnull().sum(axis=1) / len(vv_cols_raw) if vv_cols_raw else pd.Series(0, index=df.index)
miss_vh = df[vh_cols_raw].isnull().sum(axis=1) / len(vh_cols_raw) if vh_cols_raw else pd.Series(0, index=df.index)
# Keep rows where ALL metrics have <= 80% missing data
filter_mask = (miss_ndvi <= 0.8) & (miss_vv <= 0.8) & (miss_vh <= 0.8)
df_filtered = df[filter_mask].copy()
print(f"Number of samples after filtering: {len(df_filtered)}")
id_vars = ['system:index', 'class']
# Handle GEE export naming (sometimes 'class' is exported as 'first')
if 'class' not in df_filtered.columns and 'first' in df_filtered.columns:
df_filtered.rename(columns={'first': 'class'}, inplace=True)
value_vars = [col for col in df_filtered.columns if col.startswith(('NDVI', 'VV', 'VH'))]
df_long = pd.melt(df_filtered, id_vars=id_vars, value_vars=value_vars, var_name='metric_date', value_name='value')
print("Step 2: Reshaping data from wide to long format...")
# 6. SEPARATE METRIC AND DATE
df_long[['metric', 'date']] = df_long['metric_date'].str.split('_', n=1, expand=True)
df_long.drop('metric_date', axis=1, inplace=True)
print("Step 3: Separating metric type and date...")
df_processed = df_long.pivot_table(index=['system:index', 'class', 'date'], columns='metric', values='value').reset_index()
df_processed.columns.name = None
print("Step 4: Creating distinct columns for NDVI, VV, and VH...")
# 8. PROCESS DATE
df_processed['date'] = pd.to_datetime(df_processed['date'])
print("Step 5: Processing date information...")
# 9. ASSIGN REPRESENTATIVE DATE (10-Day Bins)
def get_representative_date(date_obj):
day = date_obj.day
if day <= 10: return date_obj.replace(day=5)
elif day <= 20: return date_obj.replace(day=15)
else: return date_obj.replace(day=25)
df_processed['representative_date'] = df_processed['date'].apply(get_representative_date)
print("Step 6: Assigning each row to a 10-day period representative date...")
# 10. AGGREGATE (MEDIAN)
# Group by Farm, Class, and Representative Date -> Calculate Median
aggregation_groups = df_processed.groupby(['system:index', 'class', 'representative_date'])
aggregated_df = aggregation_groups[['NDVI', 'VV', 'VH']].median().reset_index()
print("Step 7: Aggregating data and calculating medians...")
df_melted_agg = aggregated_df.melt(
id_vars=['system:index', 'class', 'representative_date'],
value_vars=['NDVI', 'VV', 'VH'],
var_name='metric',
value_name='value'
)
# Create the new column name, e.g., 'NDVI_05-10-2021'
df_melted_agg['new_col_name'] = (
df_melted_agg['metric'] + '_' +
df_melted_agg['representative_date'].dt.strftime('%d-%m-%Y')
)
# Pivot to the final wide format
df_wide = df_melted_agg.pivot_table(
index=['system:index', 'class'],
columns='new_col_name',
values='value'
).reset_index()
df_wide.columns.name = None
print("Step 8: Pivoting data into the final wide format...")
# 12. SORT COLUMNS CHRONOLOGICALLY
meta_cols = ['system:index', 'class']
metric_cols = [c for c in df_wide.columns if c not in meta_cols]
def sort_key(col_name):
parts = col_name.split('_')
metric = parts[0]
date_str = parts[1]
return (pd.to_datetime(date_str, format='%d-%m-%Y'), metric)
metric_cols.sort(key=sort_key)
final_df = df_wide[meta_cols + metric_cols]
# 13. EXPORT
print("\n--- Aggregation Complete ---")
print("Showing the first 5 rows:")
print(final_df.head(5).iloc[:, :6])
final_df.to_csv(OUTPUT_FILE, index=False)
print(f"\nSuccessfully saved to: '{OUTPUT_FILE}'")
except Exception as e:
print(f"Error: {e}")
Successfully loaded '/content/drive/MyDrive/PhD Obj1/Wheat_Test_50_50_Pivot.csv'. --- Treating 0 values as 'No Data' (NaN) ONLY for Sensor Columns --- Original data has 100 rows and 195 columns. Converting VV and VH from dB to linear scale... Converted 45 VV columns to linear scale. Converted 45 VH columns to linear scale. Step 1: Filtering samples with more than 80% missing values... Number of samples after filtering: 99 Step 2: Reshaping data from wide to long format... Step 3: Separating metric type and date... Step 4: Creating distinct columns for NDVI, VV, and VH... Step 5: Processing date information... Step 6: Assigning each row to a 10-day period representative date... Step 7: Aggregating data and calculating medians... Step 8: Pivoting data into the final wide format... --- Aggregation Complete --- Showing the first 5 rows: system:index class NDVI_05-10-2023 VH_05-10-2023 VV_05-10-2023 \ 0 1_0 1 0.516731 0.003414 0.074667 1 1_102 1 0.866221 NaN NaN 2 1_103 1 0.822888 NaN NaN 3 1_113 1 0.650379 0.014170 0.093834 4 1_114 1 0.806764 0.051765 0.104322 NDVI_15-10-2023 0 0.223833 1 0.786963 2 0.793602 3 0.500450 4 0.641726 Successfully saved to: '/content/drive/MyDrive/PhD Obj1/Wheat_Test_50_50_Aggregated.csv'
In [ ]:
import pandas as pd
# Load the file you just saved
df = pd.read_csv('/content/drive/MyDrive/PhD Obj1/Wheat_Test_50_50_Aggregated.csv')
# Count the classes
counts = df['class'].value_counts()
print("--- CLASS COUNTS ---")
print(counts)
# Show a random sample of 5 Non-Wheat (0) rows just to prove they exist
print("\n--- SAMPLE OF NON-WHEAT (0) ---")
print(df[df['class'] == 0].head(5).iloc[:, :5]) # Show first 5 cols
--- CLASS COUNTS --- class 1 50 0 49 Name: count, dtype: int64 --- SAMPLE OF NON-WHEAT (0) --- system:index class NDVI_05-10-2023 VH_05-10-2023 VV_05-10-2023 50 2_1 0 0.392121 0.020806 0.082278 51 2_10 0 0.723729 0.028377 0.093569 52 2_11 0 0.399800 0.036581 0.068647 53 2_13 0 0.488358 0.033914 0.061592 54 2_15 0 0.829457 NaN NaN
In [ ]:
import pandas as pd
import matplotlib.pyplot as plt
# 1. Load the Aggregated File
INPUT_FILE = '/content/drive/MyDrive/PhD Obj1/Wheat_Test_50_50_Aggregated.csv'
df = pd.read_csv(INPUT_FILE)
# 2. Reshape for Plotting
id_vars = ['system:index', 'class']
val_vars = [c for c in df.columns if c not in id_vars]
df_long = df.melt(id_vars=id_vars, value_vars=val_vars, var_name='col_name', value_name='value')
# 3. Extract Metric and Date
df_long[['metric', 'date_str']] = df_long['col_name'].str.split('_', n=1, expand=True)
df_long['date'] = pd.to_datetime(df_long['date_str'], format='%d-%m-%Y')
# 4. Calculate Averages per Class
# Group by Class (0/1) and Date -> Calculate Mean
df_avg = df_long.groupby(['class', 'metric', 'date'])['value'].mean().reset_index()
# 5. Plot Comparison
metrics = ['NDVI', 'VV', 'VH']
plt.figure(figsize=(18, 5))
for i, m in enumerate(metrics):
plt.subplot(1, 3, i+1)
# Plot Wheat (Green)
wheat_data = df_avg[(df_avg['metric'] == m) & (df_avg['class'] == 1)]
plt.plot(wheat_data['date'], wheat_data['value'], color='green', marker='o', linewidth=2, label='Wheat (1)')
# Plot Non-Wheat (Brown)
non_wheat_data = df_avg[(df_avg['metric'] == m) & (df_avg['class'] == 0)]
plt.plot(non_wheat_data['date'], non_wheat_data['value'], color='brown', marker='x', linestyle='--', linewidth=2, label='Non-Wheat (0)')
plt.title(f"Average {m} Profile")
plt.xlabel("Date")
plt.grid(True)
if i == 0: plt.legend()
plt.tight_layout()
plt.show()
In [ ]:
import pandas as pd
import matplotlib.pyplot as plt
import matplotlib.dates as mdates
# 1. CONFIGURATION
INPUT_FILE = '/content/drive/MyDrive/PhD Obj1/Wheat_Test_50_50_Aggregated.csv'
try:
# 2. LOAD DATA
df = pd.read_csv(INPUT_FILE)
print(f"Loaded. Shape: {df.shape}")
# 3. FILTER FOR NDVI ONLY
meta_cols = ['system:index', 'class']
ndvi_cols = [c for c in df.columns if c.startswith('NDVI_')]
df_ndvi = df[meta_cols + ndvi_cols].copy()
# 4. RESHAPE (WIDE TO LONG)
df_long = df_ndvi.melt(id_vars=['system:index', 'class'],
var_name='col_name',
value_name='NDVI')
# 5. PARSE DATES
# Extract date from "NDVI_05-10-2023"
df_long[['metric', 'date_str']] = df_long['col_name'].str.split('_', n=1, expand=True)
df_long['date'] = pd.to_datetime(df_long['date_str'], format='%d-%m-%Y')
# 6. FILTER DATE RANGE
start_date = '2023-10-01'
end_date = '2024-04-30'
mask = (df_long['date'] >= start_date) & (df_long['date'] <= end_date)
df_long = df_long.loc[mask]
df_avg = df_long.groupby(['class', 'date'])['NDVI'].mean().reset_index()
# 8. PLOT
plt.figure(figsize=(12, 6))
# --- Plot Wheat (Class 1) ---
wheat = df_avg[df_avg['class'] == 1]
plt.plot(wheat['date'], wheat['NDVI'],
color='green', marker='o', linewidth=2.5, label='Wheat (1)')
# --- Plot Non-Wheat (Class 0) ---
non_wheat = df_avg[df_avg['class'] == 0]
plt.plot(non_wheat['date'], non_wheat['NDVI'],
color='brown', marker='x', linestyle='--', linewidth=2.5, label='Non-Wheat (0)')
# Formatting
plt.title('Average NDVI Phenology (Oct - Apr)', fontsize=14)
plt.ylabel('NDVI Value', fontsize=12)
plt.xlabel('Date', fontsize=12)
plt.grid(True, linestyle=':', alpha=0.6)
plt.legend(fontsize=12)
# X-Axis Formatting (Show Month Names)
plt.gca().xaxis.set_major_formatter(mdates.DateFormatter('%b %Y'))
plt.gca().xaxis.set_major_locator(mdates.MonthLocator()) # Tick every month
plt.tight_layout()
plt.show()
print(" Visualization Complete.")
except Exception as e:
print(f"Error: {e}")
Loaded. Shape: (99, 65)
✅ Visualization Complete.
In [ ]:
import pandas as pd
import matplotlib.pyplot as plt
import matplotlib.dates as mdates
# 1. CONFIGURATION
INPUT_FILE = '/content/drive/MyDrive/PhD Obj1/Wheat_Test_50_50_Aggregated.csv'
def plot_timeseries(file_path, sample_index, plot_type):
"""
Plots a time series for a specific sample (row) and detects if it is Wheat or Non-Wheat.
"""
try:
# Load Data
df = pd.read_csv(file_path)
except FileNotFoundError:
print(f"Error: File not found at {file_path}")
return
# Validate Index
if not 0 <= sample_index < len(df):
print(f"Error: Sample index {sample_index} is out of bounds (0 to {len(df)-1}).")
return
# Validate Plot Type
valid_types = ['NDVI', 'VV', 'VH']
plot_type = plot_type.upper()
if plot_type not in valid_types:
print(f"Error: Invalid type '{plot_type}'. Choose: {valid_types}")
return
row_data = df.iloc[sample_index]
class_label = row_data['class']
class_name = "Wheat (1)" if class_label == 1 else "Non-Wheat (0)"
cols = [c for c in df.columns if c.startswith(f"{plot_type}_")]
if not cols:
print(f"No columns found for {plot_type}")
return
# Extract Dates and Values
dates = []
values = []
for col in cols:
try:
# Parse Date from Header: "NDVI_05-10-2023" -> "05-10-2023"
date_part = col.split('_')[1]
dt = pd.to_datetime(date_part, format='%d-%m-%Y')
val = row_data[col]
dates.append(dt)
values.append(val)
except Exception as e:
continue # Skip bad columns
# Sort by Date
plot_df = pd.DataFrame({'Date': dates, 'Value': values}).sort_values('Date')
# --- PLOTTING ---
plt.figure(figsize=(14, 6))
# Color logic: Green for Wheat, Brown for Non-Wheat
line_color = 'green' if class_label == 1 else 'brown'
plt.plot(plot_df['Date'], plot_df['Value'], marker='o', linestyle='-',
color=line_color, linewidth=2, label=f'Sample {sample_index}: {class_name}')
# Formatting
plt.title(f'{plot_type} Time Series for Sample {sample_index} ({class_name})', fontsize=16)
plt.ylabel(plot_type, fontsize=12)
plt.xlabel('Date', fontsize=12)
plt.grid(True, linestyle=':', alpha=0.6)
plt.legend(fontsize=12)
# Date Formatting on Axis
plt.gca().xaxis.set_major_formatter(mdates.DateFormatter('%b %Y'))
plt.gca().xaxis.set_major_locator(mdates.MonthLocator())
plt.xticks(rotation=45)
plt.tight_layout()
plt.show()
# Example 1: Look at a WHEAT sample (usually indices 0-49)
print("Plotting a Wheat Sample (Index 2)...")
plot_timeseries(INPUT_FILE, sample_index=2, plot_type='NDVI')
plot_timeseries(INPUT_FILE, sample_index=2, plot_type='VH')
# Example 2: Look at a NON-WHEAT sample (usually indices 50-99)
print("\nPlotting a Non-Wheat Sample (Index 52)...")
plot_timeseries(INPUT_FILE, sample_index=52, plot_type='NDVI')
Plotting a Wheat Sample (Index 2)...
Plotting a Non-Wheat Sample (Index 52)...
In [ ]:
import pandas as pd
import numpy as np
from scipy.interpolate import interp1d
from scipy.signal import savgol_filter
from pathlib import Path
INPUT_FILE = '/content/drive/MyDrive/PhD Obj1/Wheat_Test_50_50_Aggregated.csv'
OUTPUT_FILE = "/content/processed_time_series_15day_savgol.csv"
SAVGOL_WINDOW = 5 # Must be odd
SAVGOL_POLY = 2 # Must be less than window
def load_and_prep_data(filepath: Path) -> pd.DataFrame:
"""
Loads the wide-format CSV and converts it to a long format.
It also parses dates and metrics.
"""
if not filepath.exists():
print(f"Error: Input file not found at {filepath}")
return pd.DataFrame()
print(f"Loading data from {filepath}...")
df = pd.read_csv(filepath)
df_long = pd.melt(df, id_vars=['system:index', 'class'], var_name='metric_date', value_name='value')
# 2. Drop rows where value is NaN (these are not actual acquisitions)
df_long = df_long.dropna(subset=['value'])
# 3. Split 'metric_date' into 'metric' and 'date'
try:
split_cols = df_long['metric_date'].str.split('_', n=1, expand=True)
df_long['metric'] = split_cols[0]
df_long['date'] = pd.to_datetime(split_cols[1], format='%d-%m-%Y')
except Exception as e:
print(f"Error parsing column names. Ensure they are in 'METRIC_DD-MM-YYYY' format.")
print(f"Problematic column name might be: {df_long['metric_date'].iloc[0]}")
print(f"Error details: {e}")
return pd.DataFrame()
# 4. Filter for only the desired metrics
df_long = df_long[df_long['metric'].isin(['NDVI', 'VV', 'VH'])]
print("Applying spike filter to NDVI data (diff > 0.4)...")
# Separate NDVI from VV/VH
ndvi_mask = df_long['metric'] == 'NDVI'
df_ndvi = df_long[ndvi_mask].copy()
df_other = df_long[~ndvi_mask]
if not df_ndvi.empty:
# Sort by point and then date to prepare for .diff()
df_ndvi = df_ndvi.sort_values(by=['system:index', 'class', 'date'])
df_ndvi['value_diff'] = df_ndvi.groupby(['system:index', 'class'])['value'].diff()
# Find rows where the absolute difference is > 0.4
spike_mask = df_ndvi['value_diff'].abs() > 0.4
df_ndvi.loc[spike_mask, 'value'] = np.nan
print(f"Set {spike_mask.sum()} NDVI values to NaN due to > 0.4 difference.")
# Recombine the filtered NDVI data with the VV/VH data
df_long = pd.concat([df_ndvi.drop(columns=['value_diff']), df_other])
print("Data loading and preparation complete.")
return df_long.drop(columns=['metric_date'])
def get_common_date_range(df_long: pd.DataFrame) -> (pd.Timestamp, pd.Timestamp):
"""
Finds the common date range (latest start, earliest end)
across all three metrics.
"""
metric_ranges = df_long.groupby('metric')['date'].agg(['min', 'max'])
common_start = metric_ranges['min'].max()
common_end = metric_ranges['max'].min()
print(f"Common date range found: {common_start.date()} to {common_end.date()}")
return common_start, common_end
def process_time_series(df_long: pd.DataFrame, common_start: pd.Timestamp,
common_end: pd.Timestamp,
window: int, poly: int) -> pd.DataFrame:
"""
Interpolates, resamples, and smooths the time series for each point.
"""
print("Starting time-series processing (interpolation, resampling, smoothing)...")
# 1. Define target dates for resampling (1st and 16th of each month)
target_dates_raw = []
# Start from the 1st of the common_start month
current_date = pd.Timestamp(year=common_start.year, month=common_start.month, day=15)
while current_date <= common_end:
# Add the 1st of the month if it's within the common range
if current_date >= common_start:
target_dates_raw.append(current_date)
# Check the 16th of the month
date_16th = pd.Timestamp(year=current_date.year, month=current_date.month, day=15)
# Add the 16th if it's within the common range
if date_16th >= common_start and date_16th <= common_end:
target_dates_raw.append(date_16th)
# Move to the 1st of the next month
current_date = current_date + pd.offsets.MonthBegin(1)
# Ensure dates are sorted and unique
target_dates = pd.DatetimeIndex(sorted(list(set(target_dates_raw))))
print(target_dates)
if len(target_dates) == 0:
print("Warning: No target dates (1st or 16th) fall within the common date range.")
return pd.DataFrame()
print(f"Resampling to {len(target_dates)} target dates (1st & 16th of month)...")
target_days = (target_dates - common_start).days
results = []
grouped_by_point = df_long.groupby(['system:index', 'class'])
# Unpack both keys
for (point_id, class_val), point_data in grouped_by_point:
# Process each metric for the current point
for metric in ['NDVI', 'VV', 'VH']:
metric_data_raw = point_data[point_data['metric'] == metric].sort_values('date')
# Drop NaNs (from spike filter) before interpolation ---
metric_data = metric_data_raw.dropna(subset=['value'])
# We need at least 2 points to interpolate
if len(metric_data) < 2:
# Not enough data, fill with NaNs
smoothed_values = [np.nan] * len(target_dates)
else:
# Convert original dates to relative numerical values (days from start)
current_days = (metric_data['date'] - common_start).dt.days
current_values = metric_data['value'].values
# 2. Create interpolation function
f_interp = interp1d(current_days, current_values, kind='linear', fill_value='extrapolate')
# 3. Resample/Interpolate at target dates
resampled_values = f_interp(target_days)
# 4. Smooth the resampled data
# Ensure window size is not larger than the data itself
safe_window = min(window, len(resampled_values))
# Ensure window is odd
if safe_window % 2 == 0:
safe_window -= 1
# Ensure polyorder is less than the (safe) window
safe_poly = min(poly, safe_window - 1)
if safe_window > safe_poly and safe_poly >= 0:
smoothed_values = savgol_filter(resampled_values, safe_window, safe_poly)
else:
smoothed_values = resampled_values
# Store the results
for i, date in enumerate(target_dates):
results.append({
'system:index': point_id,
'class': class_val, # Keep the class
'date': date,
'metric': metric,
'value': smoothed_values[i]
})
print("Time-series processing complete.")
return pd.DataFrame(results)
def format_for_output(results_df: pd.DataFrame) -> pd.DataFrame:
"""
Converts the long-format results back to a wide format for saving.
"""
print("Formatting data for output...")
# Create the 'METRIC_DD-MM-YYYY' column name
results_df['col_name'] = results_df['metric'] + '_' + results_df['date'].dt.strftime('%d-%m-%Y')
wide_df = results_df.pivot(index=['system:index', 'class'], columns='col_name', values='value')
# Clean up for CSV export
wide_df = wide_df.reset_index()
wide_df.columns.name = None
# Sort the columns in ascending dates of NDVI, VV and VH
ndvi_cols = [col for col in wide_df.columns if col.startswith('NDVI_')]
ndvi_cols = sorted(ndvi_cols, key=lambda x: pd.to_datetime(x.split('_')[1], format='%d-%m-%Y'))
vv_cols = [col for col in wide_df.columns if col.startswith('VV_')]
vv_cols = sorted(vv_cols, key=lambda x: pd.to_datetime(x.split('_')[1], format='%d-%m-%Y'))
vh_cols = [col for col in wide_df.columns if col.startswith('VH_')]
vh_cols = sorted(vh_cols, key=lambda x: pd.to_datetime(x.split('_')[1], format='%d-%m-%Y'))
sorted_cols = ndvi_cols + vv_cols + vh_cols
# LOGIC CHANGE: Include 'class' in final output columns
wide_df = wide_df[['system:index', 'class'] + sorted_cols]
return wide_df
# Ensure window is odd, default to 5 if not
if SAVGOL_WINDOW % 2 == 0:
print(f"Warning: SAVGOL_WINDOW was even ({SAVGOL_WINDOW}), setting to {SAVGOL_WINDOW + 1}.")
SAVGOL_WINDOW += 1
if SAVGOL_POLY >= SAVGOL_WINDOW:
print(f"Warning: SAVGOL_POLY ({SAVGOL_POLY}) must be less than SAVGOL_WINDOW ({SAVGOL_WINDOW}).")
# Adjust polyorder to be valid
SAVGOL_POLY = max(0, SAVGOL_WINDOW - 2) # e.g., if window is 3, poly becomes 1
print(f"Adjusting SAVGOL_POLY to {SAVGOL_POLY}.")
input_path = Path(INPUT_FILE)
output_path = Path(OUTPUT_FILE)
df_long = load_and_prep_data(input_path)
if df_long.empty:
print("Processing stopped due to errors in loading data.")
common_start, common_end = get_common_date_range(df_long)
print(common_end, common_start)
if pd.isna(common_start) or pd.isna(common_end):
print("Error: Could not determine a valid common date range. Check your input data.")
if common_start > common_end:
print(f"Error: Common start date ({common_start.date()}) is after common end date ({common_end.date()}).")
print("This can happen if the time series for different metrics do not overlap.")
results_df = process_time_series(df_long, common_start, common_end,
window=SAVGOL_WINDOW,
poly=SAVGOL_POLY)
if results_df.empty:
print("No results were generated. Check intermediate steps.")
output_df = format_for_output(results_df)
output_df.to_csv(output_path, index=False)
print(f"\nSuccessfully processed data and saved to: {output_path}")
# --- PLOTTING SECTION (Visual Verification) ---
import matplotlib.pyplot as plt
from IPython.display import display
# Load DataFrames for Display
df_initial = pd.read_csv(INPUT_FILE)
df_processed = pd.read_csv(OUTPUT_FILE)
print("Initial Data Head:")
display(df_initial.head())
print("Processed Data Head:")
display(df_processed.head())
# Prepare Long Formats for Plotting
# LOGIC CHANGE: Melt with class
df_initial_long = pd.melt(df_initial, id_vars=['system:index', 'class'], var_name='metric_date', value_name='value')
df_initial_long = df_initial_long.dropna(subset=['value'])
split_cols_initial = df_initial_long['metric_date'].str.split('_', n=1, expand=True)
df_initial_long['metric'] = split_cols_initial[0]
df_initial_long['date'] = pd.to_datetime(split_cols_initial[1], format='%d-%m-%Y')
df_initial_long = df_initial_long[df_initial_long['metric'].isin(['NDVI', 'VV', 'VH'])].drop(columns=['metric_date'])
# LOGIC CHANGE: Melt with class
df_processed_long = pd.melt(df_processed, id_vars=['system:index', 'class'], var_name='metric_date', value_name='value')
split_cols_processed = df_processed_long['metric_date'].str.split('_', n=1, expand=True)
df_processed_long['metric'] = split_cols_processed[0]
df_processed_long['date'] = pd.to_datetime(split_cols_processed[1], format='%d-%m-%Y')
df_processed_long = df_processed_long[df_processed_long['metric'].isin(['NDVI', 'VV', 'VH'])].drop(columns=['metric_date'])
# Select a representative point to plot (e.g., the first point in the list)
plot_index = 0
if not df_initial_long.empty:
point_id = df_initial_long['system:index'].unique()[plot_index]
# Get class for title
class_val = df_initial_long[df_initial_long['system:index'] == point_id]['class'].iloc[0]
# Filter for the selected point
df_initial_point = df_initial_long[df_initial_long['system:index'] == point_id]
df_processed_point = df_processed_long[df_processed_long['system:index'] == point_id]
# Plot metrics
metrics = ['NDVI', 'VV', 'VH']
for metric in metrics:
plt.figure(figsize=(12, 6))
df_init = df_initial_point[df_initial_point['metric'] == metric].sort_values('date')
df_proc = df_processed_point[df_processed_point['metric'] == metric].sort_values('date')
plt.plot(df_init['date'], df_init['value'], label='Initial (Raw)', marker='o', linestyle='--', alpha=0.6)
plt.plot(df_proc['date'], df_proc['value'], label='Processed (Savgol)', marker='x', linestyle='-', linewidth=2)
plt.xlabel('Date')
plt.ylabel('Value')
plt.title(f'{metric} Time Series for Point: {point_id} (Class: {class_val})')
plt.legend()
plt.grid(True)
plt.tight_layout()
plt.show()
else:
print("No data available to plot.")
Loading data from /content/drive/MyDrive/PhD Obj1/Wheat_Test_50_50_Aggregated.csv...
Applying spike filter to NDVI data (diff > 0.4)...
Set 69 NDVI values to NaN due to > 0.4 difference.
Data loading and preparation complete.
Common date range found: 2023-10-05 to 2024-04-25
2024-04-25 00:00:00 2023-10-05 00:00:00
Starting time-series processing (interpolation, resampling, smoothing)...
DatetimeIndex(['2023-10-15', '2023-11-01', '2023-11-15', '2023-12-01',
'2023-12-15', '2024-01-01', '2024-01-15', '2024-02-01',
'2024-02-15', '2024-03-01', '2024-03-15', '2024-04-01',
'2024-04-15'],
dtype='datetime64[ns]', freq=None)
Resampling to 13 target dates (1st & 16th of month)...
Time-series processing complete.
Formatting data for output...
Successfully processed data and saved to: /content/processed_time_series_15day_savgol.csv
Initial Data Head:
system:index class NDVI_05-10-2023 VH_05-10-2023 VV_05-10-2023 \ 0 1_0 1 0.516731 0.003414 0.074667 1 1_102 1 0.866221 NaN NaN 2 1_103 1 0.822888 NaN NaN 3 1_113 1 0.650379 0.014170 0.093834 4 1_114 1 0.806764 0.051765 0.104322 NDVI_15-10-2023 VH_15-10-2023 VV_15-10-2023 NDVI_25-10-2023 \ 0 0.223833 NaN NaN 0.198677 1 0.786963 NaN NaN 0.532561 2 0.793602 NaN NaN 0.563791 3 0.500450 NaN NaN 0.297380 4 0.641726 NaN NaN 0.300372 VH_25-10-2023 ... VV_25-03-2024 NDVI_05-04-2024 VH_05-04-2024 \ 0 0.015356 ... 0.022560 0.570506 0.015989 1 NaN ... NaN 0.334932 0.006866 2 NaN ... NaN 0.565001 0.024644 3 0.015145 ... 0.066859 0.389333 0.011311 4 0.015908 ... 0.057380 0.268726 0.039646 VV_05-04-2024 NDVI_15-04-2024 VH_15-04-2024 VV_15-04-2024 \ 0 0.027344 0.236910 0.002676 0.036810 1 0.057968 0.191977 0.009740 0.075092 2 0.086908 0.254568 0.024074 0.017694 3 0.048268 0.240856 0.025576 0.053543 4 0.103278 0.177732 0.007771 0.019804 NDVI_25-04-2024 VH_25-04-2024 VV_25-04-2024 0 0.175030 NaN NaN 1 0.165339 0.000809 0.056832 2 0.184999 0.011990 0.021885 3 0.181155 NaN NaN 4 0.132486 NaN NaN [5 rows x 65 columns]
Processed Data Head:
system:index class NDVI_15-10-2023 NDVI_01-11-2023 NDVI_15-11-2023 \ 0 1_0 1 0.248812 0.098842 0.086786 1 1_102 1 0.786483 0.366661 0.152636 2 1_103 1 0.788845 0.340417 0.148498 3 1_113 1 0.502320 0.228650 0.163139 4 1_114 1 0.623343 0.239882 0.117541 NDVI_01-12-2023 NDVI_15-12-2023 NDVI_01-01-2024 NDVI_15-01-2024 \ 0 0.212738 0.451970 0.655217 0.771944 1 0.190022 0.265944 0.239125 0.254767 2 0.256439 0.483316 0.658327 0.772061 3 0.355453 0.560389 0.682171 0.721780 4 0.315329 0.616088 0.788914 0.859648 NDVI_01-02-2024 ... VH_01-12-2023 VH_15-12-2023 VH_01-01-2024 \ 0 0.861885 ... 0.007998 0.023312 0.034682 1 0.436286 ... 0.007155 0.009002 0.010580 2 0.850225 ... 0.019124 0.026643 0.024349 3 0.811168 ... 0.020166 0.017314 0.018407 4 0.881663 ... 0.017071 0.019932 0.014118 VH_15-01-2024 VH_01-02-2024 VH_15-02-2024 VH_01-03-2024 VH_15-03-2024 \ 0 0.027533 0.015077 0.006399 0.005679 0.009994 1 0.009250 0.007066 0.007278 0.010729 0.011894 2 0.014956 0.008669 0.015916 0.017987 0.020005 3 0.019663 0.016511 0.009741 0.004797 0.005360 4 0.008902 0.010668 0.013941 0.011766 0.014793 VH_01-04-2024 VH_15-04-2024 0 0.008767 0.005055 1 0.011332 0.008786 2 0.021936 0.023150 3 0.012403 0.025455 4 0.014264 0.014055 [5 rows x 41 columns]
In [ ]:
# --- REVISED PLOTTING SECTION (Plots BOTH Wheat and Non-Wheat) ---
import matplotlib.pyplot as plt
from IPython.display import display
# 1. Load Data
df_initial = pd.read_csv(INPUT_FILE)
df_processed = pd.read_csv(OUTPUT_FILE)
print("Processing Complete. Generating comparison plots...")
# 2. Reshape Data for Plotting (Standard Melt)
def prepare_plot_data(df):
df_long = pd.melt(df, id_vars=['system:index', 'class'], var_name='metric_date', value_name='value')
df_long = df_long.dropna(subset=['value'])
split = df_long['metric_date'].str.split('_', n=1, expand=True)
df_long['metric'] = split[0]
df_long['date'] = pd.to_datetime(split[1], format='%d-%m-%Y')
return df_long[df_long['metric'].isin(['NDVI', 'VV', 'VH'])].drop(columns=['metric_date'])
df_init_long = prepare_plot_data(df_initial)
df_proc_long = prepare_plot_data(df_processed)
# 3. Find One Example ID for Each Class
# Get list of IDs for Class 1 (Wheat)
wheat_ids = df_init_long[df_init_long['class'] == 1]['system:index'].unique()
# Get list of IDs for Class 0 (Non-Wheat)
non_wheat_ids = df_init_long[df_init_long['class'] == 0]['system:index'].unique()
# Pick the first available ID from each list
examples_to_plot = []
if len(wheat_ids) > 0:
examples_to_plot.append((wheat_ids[0], "Wheat (1)"))
if len(non_wheat_ids) > 0:
examples_to_plot.append((non_wheat_ids[0], "Non-Wheat (0)"))
# 4. Generate Plots
metrics = ['NDVI', 'VV', 'VH']
for point_id, label in examples_to_plot:
print(f"\n--- Visualizing {label} | Farm ID: {point_id} ---")
# Filter data for this specific farm
farm_init = df_init_long[df_init_long['system:index'] == point_id]
farm_proc = df_proc_long[df_proc_long['system:index'] == point_id]
plt.figure(figsize=(18, 5))
for i, metric in enumerate(metrics):
plt.subplot(1, 3, i+1)
# Get data for metric
d_raw = farm_init[farm_init['metric'] == metric].sort_values('date')
d_smooth = farm_proc[farm_proc['metric'] == metric].sort_values('date')
# Plot Raw (Dashed)
plt.plot(d_raw['date'], d_raw['value'], label='Raw Input',
color='gray', linestyle='--', marker='o', alpha=0.5)
# Plot Smoothed (Solid)
color = 'green' if 'Wheat' in label else 'brown'
plt.plot(d_smooth['date'], d_smooth['value'], label='Smoothed',
color=color, linestyle='-', linewidth=2, marker='x')
plt.title(f"{metric}")
plt.xlabel("Date")
plt.xticks(rotation=45)
plt.grid(True, linestyle=':', alpha=0.6)
if i == 0: plt.legend()
plt.suptitle(f"Processing Result for {label}", fontsize=16)
plt.tight_layout()
plt.show()
Processing Complete. Generating comparison plots... --- Visualizing Wheat (1) | Farm ID: 1_0 ---
--- Visualizing Non-Wheat (0) | Farm ID: 2_1 ---
In [ ]:
from google.colab import drive
drive.mount('/content/drive')
Mounted at /content/drive
In [ ]:
import pandas as pd
import numpy as np
from scipy.interpolate import interp1d
from scipy.signal import savgol_filter
from pathlib import Path
INPUT_FILE = '/content/drive/MyDrive/PhD Obj1/Wheat_Test_50_50_Aggregated.csv'
OUTPUT_FILE = "/content/processed_time_series_15day_savgol.csv"
SAVGOL_WINDOW = 5 # Must be odd
SAVGOL_POLY = 2 # Must be less than window
def load_and_prep_data(filepath: Path) -> pd.DataFrame:
"""
Loads the wide-format CSV and converts it to a long format.
It also parses dates and metrics.
"""
if not filepath.exists():
print(f"Error: Input file not found at {filepath}")
return pd.DataFrame()
print(f"Loading data from {filepath}...")
df = pd.read_csv(filepath)
df_long = pd.melt(df, id_vars=['system:index', 'class'], var_name='metric_date', value_name='value')
# 2. Drop rows where value is NaN (these are not actual acquisitions)
df_long = df_long.dropna(subset=['value'])
# 3. Split 'metric_date' into 'metric' and 'date'
try:
split_cols = df_long['metric_date'].str.split('_', n=1, expand=True)
df_long['metric'] = split_cols[0]
df_long['date'] = pd.to_datetime(split_cols[1], format='%d-%m-%Y')
except Exception as e:
print(f"Error parsing column names. Ensure they are in 'METRIC_DD-MM-YYYY' format.")
print(f"Problematic column name might be: {df_long['metric_date'].iloc[0]}")
print(f"Error details: {e}")
return pd.DataFrame()
# 4. Filter for only the desired metrics
df_long = df_long[df_long['metric'].isin(['NDVI', 'VV', 'VH'])]
print("Applying spike filter to NDVI data (diff > 0.4)...")
# Separate NDVI from VV/VH
ndvi_mask = df_long['metric'] == 'NDVI'
df_ndvi = df_long[ndvi_mask].copy()
df_other = df_long[~ndvi_mask]
if not df_ndvi.empty:
# Sort by point and then date to prepare for .diff()
df_ndvi = df_ndvi.sort_values(by=['system:index', 'class', 'date'])
df_ndvi['value_diff'] = df_ndvi.groupby(['system:index', 'class'])['value'].diff()
# Find rows where the absolute difference is > 0.4
spike_mask = df_ndvi['value_diff'].abs() > 0.4
df_ndvi.loc[spike_mask, 'value'] = np.nan
print(f"Set {spike_mask.sum()} NDVI values to NaN due to > 0.4 difference.")
# Recombine the filtered NDVI data with the VV/VH data
df_long = pd.concat([df_ndvi.drop(columns=['value_diff']), df_other])
print("Data loading and preparation complete.")
return df_long.drop(columns=['metric_date'])
def get_common_date_range(df_long: pd.DataFrame) -> (pd.Timestamp, pd.Timestamp):
"""
Finds the common date range (latest start, earliest end)
across all three metrics.
"""
metric_ranges = df_long.groupby('metric')['date'].agg(['min', 'max'])
common_start = metric_ranges['min'].max()
common_end = metric_ranges['max'].min()
print(f"Common date range found: {common_start.date()} to {common_end.date()}")
return common_start, common_end
def process_time_series(df_long: pd.DataFrame, common_start: pd.Timestamp,
common_end: pd.Timestamp,
window: int, poly: int) -> pd.DataFrame:
"""
Interpolates, resamples, and smooths the time series for each point.
"""
print("Starting time-series processing (interpolation, resampling, smoothing)...")
# 1. Define target dates for resampling (1st and 16th of each month)
target_dates_raw = []
# Start from the 1st of the common_start month
current_date = pd.Timestamp(year=common_start.year, month=common_start.month, day=15)
while current_date <= common_end:
# Add the 1st of the month if it's within the common range
if current_date >= common_start:
target_dates_raw.append(current_date)
# Check the 16th of the month
date_16th = pd.Timestamp(year=current_date.year, month=current_date.month, day=15)
# Add the 16th if it's within the common range
if date_16th >= common_start and date_16th <= common_end:
target_dates_raw.append(date_16th)
# Move to the 1st of the next month
current_date = current_date + pd.offsets.MonthBegin(1)
# Ensure dates are sorted and unique
target_dates = pd.DatetimeIndex(sorted(list(set(target_dates_raw))))
print(target_dates)
if len(target_dates) == 0:
print("Warning: No target dates (1st or 16th) fall within the common date range.")
return pd.DataFrame()
print(f"Resampling to {len(target_dates)} target dates (1st & 16th of month)...")
target_days = (target_dates - common_start).days
results = []
grouped_by_point = df_long.groupby(['system:index', 'class'])
# Unpack both keys
for (point_id, class_val), point_data in grouped_by_point:
# Process each metric for the current point
for metric in ['NDVI', 'VV', 'VH']:
metric_data_raw = point_data[point_data['metric'] == metric].sort_values('date')
# Drop NaNs (from spike filter) before interpolation ---
metric_data = metric_data_raw.dropna(subset=['value'])
# We need at least 2 points to interpolate
if len(metric_data) < 2:
# Not enough data, fill with NaNs
smoothed_values = [np.nan] * len(target_dates)
else:
# Convert original dates to relative numerical values (days from start)
current_days = (metric_data['date'] - common_start).dt.days
current_values = metric_data['value'].values
# 2. Create interpolation function
f_interp = interp1d(current_days, current_values, kind='linear', fill_value='extrapolate')
# 3. Resample/Interpolate at target dates
resampled_values = f_interp(target_days)
# 4. Smooth the resampled data
# Ensure window size is not larger than the data itself
safe_window = min(window, len(resampled_values))
# Ensure window is odd
if safe_window % 2 == 0:
safe_window -= 1
# Ensure polyorder is less than the (safe) window
safe_poly = min(poly, safe_window - 1)
if safe_window > safe_poly and safe_poly >= 0:
smoothed_values = savgol_filter(resampled_values, safe_window, safe_poly)
else:
smoothed_values = resampled_values
# Store the results
for i, date in enumerate(target_dates):
results.append({
'system:index': point_id,
'class': class_val, # Keep the class
'date': date,
'metric': metric,
'value': smoothed_values[i]
})
print("Time-series processing complete.")
return pd.DataFrame(results)
def format_for_output(results_df: pd.DataFrame) -> pd.DataFrame:
"""
Converts the long-format results back to a wide format for saving.
"""
print("Formatting data for output...")
# Create the 'METRIC_DD-MM-YYYY' column name
results_df['col_name'] = results_df['metric'] + '_' + results_df['date'].dt.strftime('%d-%m-%Y')
wide_df = results_df.pivot(index=['system:index', 'class'], columns='col_name', values='value')
# Clean up for CSV export
wide_df = wide_df.reset_index()
wide_df.columns.name = None
# Sort the columns in ascending dates of NDVI, VV and VH
ndvi_cols = [col for col in wide_df.columns if col.startswith('NDVI_')]
ndvi_cols = sorted(ndvi_cols, key=lambda x: pd.to_datetime(x.split('_')[1], format='%d-%m-%Y'))
vv_cols = [col for col in wide_df.columns if col.startswith('VV_')]
vv_cols = sorted(vv_cols, key=lambda x: pd.to_datetime(x.split('_')[1], format='%d-%m-%Y'))
vh_cols = [col for col in wide_df.columns if col.startswith('VH_')]
vh_cols = sorted(vh_cols, key=lambda x: pd.to_datetime(x.split('_')[1], format='%d-%m-%Y'))
sorted_cols = ndvi_cols + vv_cols + vh_cols
# LOGIC CHANGE: Include 'class' in final output columns
wide_df = wide_df[['system:index', 'class'] + sorted_cols]
return wide_df
# Ensure window is odd, default to 5 if not
if SAVGOL_WINDOW % 2 == 0:
print(f"Warning: SAVGOL_WINDOW was even ({SAVGOL_WINDOW}), setting to {SAVGOL_WINDOW + 1}.")
SAVGOL_WINDOW += 1
if SAVGOL_POLY >= SAVGOL_WINDOW:
print(f"Warning: SAVGOL_POLY ({SAVGOL_POLY}) must be less than SAVGOL_WINDOW ({SAVGOL_WINDOW}).")
# Adjust polyorder to be valid
SAVGOL_POLY = max(0, SAVGOL_WINDOW - 2) # e.g., if window is 3, poly becomes 1
print(f"Adjusting SAVGOL_POLY to {SAVGOL_POLY}.")
input_path = Path(INPUT_FILE)
output_path = Path(OUTPUT_FILE)
df_long = load_and_prep_data(input_path)
if df_long.empty:
print("Processing stopped due to errors in loading data.")
common_start, common_end = get_common_date_range(df_long)
print(common_end, common_start)
if pd.isna(common_start) or pd.isna(common_end):
print("Error: Could not determine a valid common date range. Check your input data.")
if common_start > common_end:
print(f"Error: Common start date ({common_start.date()}) is after common end date ({common_end.date()}).")
print("This can happen if the time series for different metrics do not overlap.")
results_df = process_time_series(df_long, common_start, common_end,
window=SAVGOL_WINDOW,
poly=SAVGOL_POLY)
if results_df.empty:
print("No results were generated. Check intermediate steps.")
output_df = format_for_output(results_df)
output_df.to_csv(output_path, index=False)
print(f"\nSuccessfully processed data and saved to: {output_path}")
Loading data from /content/drive/MyDrive/PhD Obj1/Wheat_Test_50_50_Aggregated.csv...
Applying spike filter to NDVI data (diff > 0.4)...
Set 69 NDVI values to NaN due to > 0.4 difference.
Data loading and preparation complete.
Common date range found: 2023-10-05 to 2024-04-25
2024-04-25 00:00:00 2023-10-05 00:00:00
Starting time-series processing (interpolation, resampling, smoothing)...
DatetimeIndex(['2023-10-15', '2023-11-01', '2023-11-15', '2023-12-01',
'2023-12-15', '2024-01-01', '2024-01-15', '2024-02-01',
'2024-02-15', '2024-03-01', '2024-03-15', '2024-04-01',
'2024-04-15'],
dtype='datetime64[ns]', freq=None)
Resampling to 13 target dates (1st & 16th of month)...
Time-series processing complete.
Formatting data for output...
Successfully processed data and saved to: /content/processed_time_series_15day_savgol.csv
In [ ]:
Loading processed data from /content/processed_time_series_15day_savgol.csv... ✅ Data Loaded. Shape: (99, 41) Plotting 10 Wheat samples vs 10 Non-Wheat samples.
In [ ]:
import pandas as pd
import plotly.graph_objects as go
from plotly.subplots import make_subplots
# 1. CONFIGURATION
INPUT_FILE = '/content/processed_time_series_15day_savgol.csv'
try:
print(f"Loading processed data from {INPUT_FILE}...")
df = pd.read_csv(INPUT_FILE)
print(f" Data Loaded. Shape: {df.shape}")
# 2. PREPARE DATA (Melt & Parse)
# Convert Wide -> Long
df_long = pd.melt(df, id_vars=['system:index', 'class'], var_name='metric_date', value_name='value')
# Parse Dates
split_cols = df_long['metric_date'].str.split('_', n=1, expand=True)
df_long['metric'] = split_cols[0]
df_long['date'] = pd.to_datetime(split_cols[1], format='%d-%m-%Y')
# Filter: NDVI Only, Sort by Date
df_ndvi = df_long[df_long['metric'] == 'NDVI'].sort_values('date')
# 3. SELECT 50 SAMPLES PER CLASS
wheat_ids = df_ndvi[df_ndvi['class'] == 1]['system:index'].unique()[:50]
non_wheat_ids = df_ndvi[df_ndvi['class'] == 0]['system:index'].unique()[:50]
print(f"Visualizing {len(wheat_ids)} Wheat vs. {len(non_wheat_ids)} Non-Wheat samples.")
fig = make_subplots(
rows=1, cols=2,
subplot_titles=(f"WHEAT (Class 1) - {len(wheat_ids)} Samples",
f"NON-WHEAT (Class 0) - {len(non_wheat_ids)} Samples"),
shared_yaxes=True,
horizontal_spacing=0.05
)
# --- HELPER TO ADD LINES ---
def add_traces(id_list, class_name, col_idx, color_hex):
# 1. Add Individual Farm Lines
for pid in id_list:
farm_data = df_ndvi[df_ndvi['system:index'] == pid]
fig.add_trace(
go.Scatter(
x=farm_data['date'],
y=farm_data['value'],
mode='lines+markers',
name=str(pid), # Index Name appears in Legend
line=dict(color=color_hex, width=1),
opacity=0.4,
# HOVER TEMPLATE: This is what you see when you hover
hovertemplate=(
f"<b>ID: {pid}</b><br>" +
"Date: %{x|%d-%b-%Y}<br>" +
"NDVI: %{y:.2f}<br>" +
f"Class: {class_name}" +
"<extra></extra>" # Hides the secondary box
),
showlegend=False # Hide individual farms from legend to keep it clean
),
row=1, col=col_idx
)
# 2. Add AVERAGE Trend Line (Thick & Distinct)
class_val = 1 if class_name == "Wheat" else 0
avg_data = df_ndvi[df_ndvi['class'] == class_val].groupby('date')['value'].mean().reset_index()
fig.add_trace(
go.Scatter(
x=avg_data['date'],
y=avg_data['value'],
mode='lines',
name=f'<b>AVERAGE {class_name}</b>',
line=dict(color='black', width=4, dash='solid'),
hovertemplate="<b>AVERAGE TREND</b><br>Date: %{x}<br>NDVI: %{y:.2f}<extra></extra>"
),
row=1, col=col_idx
)
# --- ADD TRACES ---
# Wheat (Column 1, Green)
add_traces(wheat_ids, "Wheat", 1, '#2ca02c') # Matplotlib Green
# Non-Wheat (Column 2, Brown)
add_traces(non_wheat_ids, "Non-Wheat", 2, '#8c564b') # Matplotlib Brown
# 5. FINALIZE LAYOUT
fig.update_layout(
title_text="<b>Interactive Smoothed NDVI Analysis</b> (Hover to Identify False Negatives)",
height=600,
template="plotly_white",
hovermode="x unified" # Shows all values for that date in a list
)
# Set Y-Axis Limits (NDVI is 0 to 1)
fig.update_yaxes(title_text="Smoothed NDVI", range=[0, 1], row=1, col=1)
fig.update_xaxes(title_text="Date", row=1, col=1)
fig.update_xaxes(title_text="Date", row=1, col=2)
fig.show()
except FileNotFoundError:
print(f" Error: File not found at {INPUT_FILE}")
except Exception as e:
print(f"Error: {e}")
Loading processed data from /content/processed_time_series_15day_savgol.csv... ✅ Data Loaded. Shape: (99, 41) Visualizing 50 Wheat vs. 49 Non-Wheat samples.