파이썬으로 표 형식 데이터 다루기

!pip install basemap 
!pip install scikeras
!pip install tensorflow

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import mean_squared_error, mean_absolute_error
import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Dropout
from tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau

# Loading Dataset
data = pd.read_csv("database.csv")
print(data.isnull())  # checking for null values

# Handling Missing Values
data=data.interpolate(method ='linear', limit_direction ='forward')
print(data)

# lets see the dataset statistic 
data.describe()

plt.figure(figsize=(10, 6))
plt.scatter(data['Longitude'], data['Latitude'], c=data['Magnitude'], cmap='viridis', s=50, alpha=0.7)
plt.colorbar(label='Magnitude')
plt.xlabel('Longitude')
plt.ylabel('Latitude')
plt.title('Earthquake Locations')
plt.show()

# Scatter plot for Magnitude and Depth
plt.figure(figsize=(10, 6))
plt.scatter(data['Magnitude'], data['Depth'], alpha=0.7)
plt.xlabel('Magnitude')
plt.ylabel('Depth')
plt.title('Magnitude vs Depth')
plt.show()

from mpl_toolkits.basemap import Basemap


m = Basemap(projection='mill',llcrnrlat=-80,urcrnrlat=80, llcrnrlon=-180,urcrnrlon=180,lat_ts=20,resolution='c')


longitudes = data["Longitude"].tolist()
latitudes = data["Latitude"].tolist()
x,y = m(longitudes,latitudes)


fig = plt.figure(figsize=(12,10))
plt.title("All affected areas")
m.plot(x, y, "o", markersize = 2, color = 'green')
m.drawcoastlines()
m.fillcontinents(color='coral',lake_color='aqua')
m.drawmapboundary()
m.drawcountries()
plt.show()

data = data[['Date', 'Time', 'Latitude', 'Longitude', 'Depth', 'Magnitude']]
data.head()

import datetime
import time


timestamp = []
for d, t in zip(data['Date'], data['Time']):
    try:
        ts = datetime.datetime.strptime(d+' '+t, '%m/%d/%Y %H:%M:%S')
        timestamp.append(time.mktime(ts.timetuple()))
    except ValueError:
        # print('ValueError')
        timestamp.append('ValueError')


timeStamp = pd.Series(timestamp)
data['Timestamp'] = timeStamp.values


final_data = data.drop(['Date', 'Time'], axis=1)
final_data = final_data[final_data.Timestamp != 'ValueError']
final_data.head()

X = final_data[['Timestamp', 'Latitude', 'Longitude']]
y = final_data[['Magnitude', 'Depth']]

from sklearn.model_selection import train_test_split
import wandb
from wandb.integration.keras import WandbEvalCallback, WandbMetricsLogger


X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)


# Normalize the data
scaler = StandardScaler()
X_train = scaler.fit_transform(X_train)
X_test = scaler.transform(X_test)

# Define the model
model = Sequential()
model.add(Dense(128, input_dim=X_train.shape[1], activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(64, activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(32, activation='relu'))
model.add(Dense(2))  # Output layer with 2 neurons for 'Magnitude' and 'Depth'


# Compile the model
model.compile(optimizer='adam', loss='mean_squared_error')


# Summary of the model
model.summary()

history = model.fit(X_train, y_train, epochs=200, batch_size=32, validation_split=0.2, verbose=1)


test_loss = model.evaluate(X_test, y_test, verbose=1) 
print(f'Test Loss: {test_loss}')


# Make predictions 
y_pred = model.predict(X_test)


# Calculate the Mean Squared Error 
mse = mean_squared_error(y_test, y_pred) print(f'Mean Squared Error: {mse}')

from tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau
wandb.init(config={"hyper": "parameter"})

early_stopping = EarlyStopping(monitor='val_loss', patience=20, restore_best_weights=True)
reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=10, min_lr=0.0001)



class WandbClfEvalCallback(WandbEvalCallback):
    def __init__(self, validation_data, data_table_columns, pred_table_columns, num_samples=100):
        super().__init__(data_table_columns, pred_table_columns)
        self.x = validation_data[0]
        self.y = validation_data[1]
        self.num_samples = num_samples

    def add_ground_truth(self, logs=None):
        for idx, (features, label) in enumerate(zip(self.x[:self.num_samples], self.y[:self.num_samples])):
            self.data_table.add_data(idx, features.tolist(), label[0], label[1])

    def add_model_predictions(self, epoch, logs=None):
        preds = self._inference()
        table_idxs = self.data_table_ref.get_index()

        for idx in table_idxs:
            pred = preds[idx]
            self.pred_table.add_data(
                epoch,
                self.data_table_ref.data[idx][0],  # features
                self.data_table_ref.data[idx][1],  # true magnitude
                self.data_table_ref.data[idx][2],  # true depth
                pred[0],  # predicted magnitude
                pred[1]   # predicted depth
            )

    def _inference(self):
        preds = []
        for features in self.x[:self.num_samples]:
            pred = self.model.predict(tf.expand_dims(features, axis=0), verbose=0)
            preds.append(pred[0])
        return preds

history = model.fit(
    X_train, y_train,
    validation_data=(X_test, y_test),
    epochs=300,
    batch_size=32,
    verbose=1,
    callbacks=[
        early_stopping,
        reduce_lr,
       WandbMetricsLogger(),
        WandbClfEvalCallback(
            validation_data=(X_test, y_test),
            data_table_columns=["idx", "features", "true_magnitude", "true_depth"],
            pred_table_columns=["epoch", "features", "true_magnitude", "true_depth", "pred_magnitude", "pred_depth"],
        ),
    ]
)