Scribd
Upload
43 views

Pinot DB Schema

The document describes setting up an Apache Pinot cluster with Docker containers for Zookeeper, Kafka, Pinot Controller, Pinot Broker and Pinot Server. It includes commands to create a Kafka topic, add a Pinot table, run a SQL query on Pinot and connect Pinot to a PostgreSQL database to store query results.

Uploaded by

Akshay Sripad
Copyright
© © All Rights Reserved
We take content rights seriously. If you suspect this is your content, claim it here.
Available Formats
Download as TXT, PDF, TXT or read online on Scribd
43 views

Pinot DB Schema

The document describes setting up an Apache Pinot cluster with Docker containers for Zookeeper, Kafka, Pinot Controller, Pinot Broker and Pinot Server. It includes commands to create a Kafka topic, add a Pinot table, run a SQL query on Pinot and connect Pinot to a PostgreSQL database to store query results.

Uploaded by

Akshay Sripad
Copyright
© © All Rights Reserved
We take content rights seriously. If you suspect this is your content, claim it here.
Available Formats
Download as TXT, PDF, TXT or read online on Scribd
You are on page 1/ 4

curl -H "Content-Type: application/json" -X POST -d "{\"sql\": \"select * from

wikipedia\"}" http://localhost:8099/query/sql

{
"schemaName": "wikipedia_user_bot_counts",
"dimensionFieldSpecs": [
{
"name": "user",
"dataType": "STRING"
},
{
"name": "bot",
"dataType": "STRING" // You can use "BOOLEAN" if "bot" represents Boolean
values
}
],
"metricFieldSpecs": [
{
"name": "record_count",
"dataType": "LONG"
}
]
}

{
"schemaName": "wikipedia_user_bot_counts",
"dimensionFieldSpecs": [
{"name": "user", "dataType": "STRING"},
{"name": "bot", "dataType": "STRING"} // Assuming you want "bot" as STRING
type
],
"metricFieldSpecs": [
{"name": "record_count", "dataType": "LONG"}
]
}

curl -H "Content-Type: application/json" -X POST


-d "{\"sql\": \"SELECT user, bot, COUNT(*) AS record_count FROM wikipedia GROUP BY
user, bot limit 100\" }" http://localhost:8099/query/sql > output.json

import subprocess
import json
import time
import psycopg2 # Assuming you're using PostgreSQL
import requests

# PostgreSQL connection parameters


db_params = {
"host": "localhost",
"port": 25432,
"database": "pino2",
"user": "user",
"password": "user",
}

conn = psycopg2.connect(**db_params)
# Function to run the Pinot query and save results to the database
def run_pinot_query_and_save():
try:
# Define the Pinot query in JSON format
pinot_query = {
"sql": "SELECT domain, count(user) AS changes FROM wikipedia GROUP BY
domain ORDER BY changes DESC LIMIT 10"
}

# Send the Pinot query to Pinot's SQL endpoint


response = requests.post(
"http://localhost:8099/query/sql",
headers={"Content-Type": "application/json"},
data=json.dumps(pinot_query),
)

# Parse the JSON response


query_result = response.json()

# Extract relevant data from the JSON response


domain = query_result['resultTable']['rows'][0][0] # Assuming you want the
first domain
changes = query_result['resultTable']['rows'][0][1] # Assuming you want
the first changes value

# Save the extracted data to the database


cursor = conn.cursor()
cursor.execute(
"INSERT INTO postg (domain, changes) VALUES (%s, %s)",
(domain, changes),
)
conn.commit()
cursor.close()

print("Query executed and result saved to the database.")


except Exception as e:
print(f"Error: {str(e)}")

# Run the query and save results every 10 seconds


while True:
run_pinot_query_and_save()
time.sleep(10) # Sleep for 10 seconds before the next query

docker exec -it kafka-wiki kafka-topics.sh --bootstrap-server localhost:9092 --


partitions 5 --topic wiki-events --create
docker exec -it monitoring-apache-pinot-master-pinot-controller-1 bin/pinot-
admin.sh AddTable -tableConfigFile /config/table.json -schemaFile
/config/schema.json -exec

curl -H "Content-Type: application/json" -X POST -d "{\"sql\": \"SELECT


oe.customer_id, oe.item,oe.location,oe.order_id,oe.quantity,oe.restaurant_id,
oe.restaurant_name, oe.ts as order_ts, ce.customer_name, ce.email, ce.phone, ce.ts
as customer_ts FROM order_events oe JOIN customer_events ce ON oe.customer_id =
ce.customer_id}" http://localhost:8099/query/sql > output.json
version: '3.7'
services:
zookeeper:
image: zookeeper:3.5.6
container_name: "zookeeper-wiki"
ports:
- "2181:2181"
environment:
ZOOKEEPER_CLIENT_PORT: 2181
ZOOKEEPER_TICK_TIME: 2000
kafka:
image: wurstmeister/kafka:latest
restart: unless-stopped
container_name: "kafka-wiki"
ports:
- "9092:9092"
expose:
- "9093"
depends_on:
- zookeeper
environment:
KAFKA_ZOOKEEPER_CONNECT: zookeeper-wiki:2181/kafka
KAFKA_BROKER_ID: 0
KAFKA_ADVERTISED_HOST_NAME: kafka-wiki
KAFKA_ADVERTISED_LISTENERS:
PLAINTEXT://kafka-wiki:9093,OUTSIDE://localhost:9092
KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9093,OUTSIDE://0.0.0.0:9092
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,OUTSIDE:PLAINTEXT
pinot-controller:
image: apachepinot/pinot:0.12.0
command: "StartController -zkAddress zookeeper-wiki:2181 -dataDir /data"
container_name: "pinot-controller-wiki"
volumes:
- ./config:/config
- ./data:/data
restart: unless-stopped
ports:
- "9000:9000"
environment:
- PINOT_OPTS=-Dcom.sun.management.jmxremote -
Dcom.sun.management.jmxremote.port=9010 -Dcom.sun.management.jmxremote.ssl=false -
Dcom.sun.management.jmxremote.authenticate=false
depends_on:
- zookeeper
pinot-broker:
image: apachepinot/pinot:0.12.0
command: "StartBroker -zkAddress zookeeper-wiki:2181"
restart: unless-stopped
container_name: "pinot-broker-wiki"
volumes:
- ./config:/config
ports:
- "8099:8099"
environment:
- PINOT_OPTS=-Dcom.sun.management.jmxremote -
Dcom.sun.management.jmxremote.port=9010 -Dcom.sun.management.jmxremote.ssl=false -
Dcom.sun.management.jmxremote.authenticate=false
depends_on:
- pinot-controller
pinot-server:
image: apachepinot/pinot:0.12.0
command: "StartServer -zkAddress zookeeper-wiki:2181"
restart: unless-stopped
container_name: "pinot-server-wiki"
volumes:
- ./config:/config
ports:
- "8098:8098"
environment:
- PINOT_OPTS=-Dcom.sun.management.jmxremote -
Dcom.sun.management.jmxremote.port=9010 -Dcom.sun.management.jmxremote.ssl=false -
Dcom.sun.management.jmxremote.authenticate=false
depends_on:
- pinot-broker

You might also like