# Run this in Jupyter Notebook cell
import pandas as pd
import json

# 1. Structured Data (CSV)
df_structured = pd.read_csv("https://raw.githubusercontent.com/datasciencedojo/datasets/master/titanic.csv")
print("Structured Data (Titanic CSV):")
print(df_structured.head(3))

# 2. Semi-structured Data (JSON)
json_data = '''
[{"name": "Alice", "age": 30, "city": "New York"},
 {"name": "Bob",   "age": 25, "city": "London", "hobbies": ["cricket","coding"]}]
'''
data = json.loads(json_data)
print("\nSemi-structured Data (JSON):")
print(pd.json_normalize(data))

# 3. Unstructured Data (Text from tweet-like)
unstructured = "Just deployed my #Spark cluster on @GoogleCloud! Loving the performance 🚀 #BigData"
print("\nUnstructured Text:")
print(unstructured)

Sources → Ingestion → Storage (Data Lake) → Processing → Serving → Consumption
         (Kafka/Flink)    (S3/GCS + Delta Lake)   (Spark/Databricks)   (BigQuery/Looker

# Lab 2: Full Spark + Delta Lake Pipeline in Colab (2025)
!pip install pyspark delta-spark -q

from pyspark.sql import SparkSession
from delta import *

# Create Spark session with Delta Lake
builder = SparkSession.builder.appName("BigDataLab") \
    .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") \
    .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog")

spark = configure_spark_with_delta_pip(builder).getOrCreate()

# Create sample data (simulating IoT sensors)
data = [
    (1, "2025-11-30 10:00:00", 23.5, "temperature"),
    (2, "2025-11-30 10:01:00", 98.0, "pressure"),
    (3, "2025-11-30 10:02:00", 45.0, "humidity")
]
df = ["device_id", "timestamp", "value", "metric"]
df = spark.createDataFrame(data, columns)

# Write as Delta Lake table (ACID transactions!)
df.write.format("delta").mode("overwrite").save("/tmp/iot_delta")

# Read back with full SQL support
delta_df = spark.read.format("delta").load("/tmp/iot_delta")
delta_df.show()

# Time travel! See previous version
spark.read.format("delta").option("versionAsOf", 0).load("/tmp/iot_delta").show()

# Run SQL
spark.sql("CREATE TABLE iot USING DELTA LOCATION '/tmp/iot_delta'")
spark.sql("SELECT * FROM iot WHERE value > 50").show()

# Delete a user's data permanently
from delta.tables import *
deltaTable = DeltaTable.forPath(spark, "/tmp/iot_delta")
deltaTable.delete("device_id = 1")  # GDPR delete request
deltaTable.toDF().show()