From d3e6cae7ea36b695835c4b1f98a0a91c7e5c6ec2 Mon Sep 17 00:00:00 2001 From: Blaise Thompson Date: Wed, 28 Oct 2020 17:44:13 -0500 Subject: initial working docker-compose --- .gitignore | 1 + README.md | 11 +++-- docker-compose.yml | 26 +++++++++++ mosquitto/acl_file | 10 +++++ mosquitto/dockerfile | 2 + mosquitto/mosquitto.conf | 3 ++ mosquitto/password_file | 3 ++ web/dockerfile | 2 + web/index.html | 102 ++++++++++++++++++++++++++++++++++++++++++ web/style.css | 18 ++++++++ write-influx.service | 12 ----- write-influx/dockerfile | 4 ++ write-influx/requirements.txt | 2 + write-influx/write_influx.py | 90 +++++++++++++++++++++++++++++++++++++ write_influx.py | 91 ------------------------------------- 15 files changed, 268 insertions(+), 109 deletions(-) create mode 100644 .gitignore create mode 100644 docker-compose.yml create mode 100644 mosquitto/acl_file create mode 100644 mosquitto/dockerfile create mode 100644 mosquitto/mosquitto.conf create mode 100644 mosquitto/password_file create mode 100644 web/dockerfile create mode 100644 web/index.html create mode 100644 web/style.css delete mode 100644 write-influx.service create mode 100644 write-influx/dockerfile create mode 100644 write-influx/requirements.txt create mode 100644 write-influx/write_influx.py delete mode 100644 write_influx.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c2de89b --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +/db diff --git a/README.md b/README.md index 3a21245..bcac4ad 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,11 @@ # mqtt -documentation and source code for departmental mqtt server +Documentation and source code for departmental mqtt server hosted at https://mosquitto.chem.wisc.edu. -hosted at https://mosquitto.chem.wisc.edu - -## operating system - -ubuntu 20.04 +To run: +``` +$ docker-compose up +``` ## mosquitto diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..46c7863 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,26 @@ +version: '3.8' + +services: + broker: + build: ./mosquitto + ports: + - 1883:1883 + db: + image: influxdb + ports: + - 8086:8086 + volumes: + - ./db:/var/lin/influxdb + write-influx: + build: ./write-influx + depends_on: + - broker + - db + web: + build: ./web + ports: + - 8080:80 + depends_on: + - broker + - db + - write-influx \ No newline at end of file diff --git a/mosquitto/acl_file b/mosquitto/acl_file new file mode 100644 index 0000000..ea24aa7 --- /dev/null +++ b/mosquitto/acl_file @@ -0,0 +1,10 @@ +topic read # + +user blaise +topic readwrite # + +user bertram +topic readwrite homie/# + +user wright +topic readwrite homie/# \ No newline at end of file diff --git a/mosquitto/dockerfile b/mosquitto/dockerfile new file mode 100644 index 0000000..272a884 --- /dev/null +++ b/mosquitto/dockerfile @@ -0,0 +1,2 @@ +FROM eclipse-mosquitto +COPY . /mosquitto/config/ \ No newline at end of file diff --git a/mosquitto/mosquitto.conf b/mosquitto/mosquitto.conf new file mode 100644 index 0000000..83ca3b0 --- /dev/null +++ b/mosquitto/mosquitto.conf @@ -0,0 +1,3 @@ +acl_file /mosquitto/config/acl_file +allow_anonymous true +password_file /mosquitto/config/password_file \ No newline at end of file diff --git a/mosquitto/password_file b/mosquitto/password_file new file mode 100644 index 0000000..351f01f --- /dev/null +++ b/mosquitto/password_file @@ -0,0 +1,3 @@ +blaise:$6$iBT2ov0mgSrG+lYu$U4yYZDLB592u48U4dBeIXCcG7gCLcF+wkoOMjtD2U0f1TmzMoUwjumO0j3qjG7qqfqLchIn1104E11OggyGZ+A== +bertram:$6$W+J9HvADoWxUYDHf$p/oYoJg7VDNwJaJtb9cdyu/m+TYzI0bhaok9jY0eqwMKGd1oP0NuOY6qJqMjklAwoggigwPxVbmHXVJrOj6uhQ== +wright:$6$AcOB+6Dp11IvqgpT$6MnWVY0E3KdMUTZXkwammbMhxeoWsKorW+bfHCulOXYL9x+Y7UG34SBQL5tPQeIMdNjyLpFaN4sF4haMm9HetA== diff --git a/web/dockerfile b/web/dockerfile new file mode 100644 index 0000000..07ec426 --- /dev/null +++ b/web/dockerfile @@ -0,0 +1,2 @@ +FROM nginx:mainline +COPY . /usr/share/nginx/html \ No newline at end of file diff --git a/web/index.html b/web/index.html new file mode 100644 index 0000000..78d52a6 --- /dev/null +++ b/web/index.html @@ -0,0 +1,102 @@ + + + + + + mosquitto + + + + + +

mosquitto.chem.wisc.edu

+ +
+ +

+This server hosts an MQTT broker for "internet of things" applications within the Department of Chemistry at the University of Wisconsin-Madison.
+In particular, this server hosts: +

+ + + +

+This server is purposefully isolated to the campus network.
+If you are reading this, congrats---you're on the network. +

+ +

+This server is maintained by Blaise Thompson and Alan Silver.
+Please contact them with any questions or concerns. +

+ +

mosquitto

+ +

+This server runs mosquitto, an MQTT broker. +Briefly, MQTT is a lightweight publish-subscribe protocol that allows many devices to communicate. +The broker provides structured topics, e.g. "sensor42/temperature" and "instrument9/opa2/wavelength". +Clients can publish topic updates. +Other clients can subscribe to be notified each time a topic is updated. +Refer to mqtt.org for more information. +

+ +

+Anyone on the campus network can subscribe to the topics served here. +There are lots of great MQTT clients you can use. +We recommend: +

+ + + +

+You'll need a password to publish to this broker. +Talk to Blaise Thompson or Alan Silver if you want to start publishing. +

+ +

influxdb

+ +

+The Chemistry Department has standardized on the homie convention. +This convention imposes a well-defined structure on top of the basic MQTT communication layer. +This structure introduces a "device" topology. + +

+This server stores all homie device history to an influx time-series database. +Anyone on the campus network can query this database. +Importantly, only homie-convention compliant MQTT topics are stored---other published topics are not recorded in any permanent way. +

+ +

+While we make every attempt to prevent data loss, please do not treat this database as an archive. +You are responsible for saving data that you care about in a more permanent way. +Device history older than two years will automatically be deleted. +

+ +

+EXAMPLE CURL +

+ +

+EXAMPLE PYTHON +

+ +

+RECOMMEND GRAFANA +

+ +
+ +

+CC0: no copyright +

+ + + diff --git a/web/style.css b/web/style.css new file mode 100644 index 0000000..4e33ceb --- /dev/null +++ b/web/style.css @@ -0,0 +1,18 @@ +a { + text-decoration: none; + color: #4271ae; +} + +body { + font-family: 'DejaVu Sans Mono', monospace; + margin: 40px auto; + min-width: 80ch; + max-width: 80ch; + line-height: 1.5; + font-size: 16px; + background-color: #ffffff; + color: #4d4d4c; + padding: 0 10px; + text-align: left; + overflow-y: scroll; +} diff --git a/write-influx.service b/write-influx.service deleted file mode 100644 index aaed2b6..0000000 --- a/write-influx.service +++ /dev/null @@ -1,12 +0,0 @@ - -[Unit] -Description=write-influx - -[Service] -Type=simple -User=blaise -ExecStart= -ExecReload=/bin/kill -HUP $MAINPID - -[Install] -WantedBy=multi-user.target diff --git a/write-influx/dockerfile b/write-influx/dockerfile new file mode 100644 index 0000000..fa509f8 --- /dev/null +++ b/write-influx/dockerfile @@ -0,0 +1,4 @@ +FROM python:3.8 +COPY . ./ +RUN pip install --no-cache-dir -r requirements.txt +CMD ["python", "-u", "write_influx.py"] diff --git a/write-influx/requirements.txt b/write-influx/requirements.txt new file mode 100644 index 0000000..84d08b3 --- /dev/null +++ b/write-influx/requirements.txt @@ -0,0 +1,2 @@ +influxdb +paho-mqtt diff --git a/write-influx/write_influx.py b/write-influx/write_influx.py new file mode 100644 index 0000000..17b4ce7 --- /dev/null +++ b/write-influx/write_influx.py @@ -0,0 +1,90 @@ +from influxdb import InfluxDBClient +import os +import paho.mqtt.client as mqtt +import time +import pathlib +import datetime +from dataclasses import dataclass, field +from typing import List, Dict + + +class Topic(dict): + + def __init__(self): + super().__init__() + self["__value__"] = None + + def __getitem__(self, key): + try: + return super().__getitem__(key) + except KeyError: + out = Topic() + self[key] = out + return out + + +homie = Topic() + + +def on_connect(client, userdata, flags, rc): + print("Connected with result code " + str(rc)) + client.subscribe("homie/#") + + +def on_message(client, userdata, msg): + topics = msg.topic.split("/") + payload = msg.payload.decode() + # fill topic tree + here = homie + for t in topics[1:]: + here = here[t] + here["__value__"] = payload + # tags + tags = {} + tags["device_id"] = topics[1] + # case of device attribute + device_attributes = ["$homie", "$name", "$state", "$nodes", "$extensions", "$implementation"] + if len(topics) == 3 and topics[2] in device_attributes: + measurement = topics[2] + fields = {"value": payload} + write_point(measurement, tags, fields) + # case of device node property + elif len(topics) == 4: + measurement = topics[3] + tags["node"] = topics[2] + datatype = homie[topics[1]][topics[2]][topics[3]]["$datatype"]["__value__"] + if datatype == "float": + fields = {"value": float(payload)} + else: + return + write_point(measurement, tags, fields) + + +influx_client = InfluxDBClient("db", 8086, 'root', 'root', "homie") +influx_client.create_database("homie") + + +def write_point(measurement, tags, fields): + json = {} + json["measurement"] = measurement + json["tags"] = tags + json["fields"] = fields + try: + influx_client.write_points([json]) + except Exception as e: + print(e) + print(json) + + + +client = mqtt.Client() +client.on_connect = on_connect +client.on_message = on_message + +client.connect("broker", 1883, 60) + +# Blocking call that processes network traffic, dispatches callbacks and +# handles reconnecting. +# Other loop*() functions are available that give a threaded interface and a +# manual interface. +client.loop_forever() diff --git a/write_influx.py b/write_influx.py deleted file mode 100644 index 240b10b..0000000 --- a/write_influx.py +++ /dev/null @@ -1,91 +0,0 @@ -from influxdb import InfluxDBClient -import os -import paho.mqtt.client as mqtt -import time -import pathlib -import datetime -from dataclasses import dataclass, field -from typing import List, Dict - - -class Topic(dict): - - def __init__(self): - super().__init__() - self["__value__"] = None - - def __getitem__(self, key): - try: - return super().__getitem__(key) - except KeyError: - out = Topic() - self[key] = out - return out - - -homie = Topic() - - -def on_connect(client, userdata, flags, rc): - print("Connected with result code " + str(rc)) - client.subscribe("homie/#") - - -def on_message(client, userdata, msg): - topics = msg.topic.split("/") - payload = msg.payload.decode() - # fill topic tree - here = homie - for t in topics[1:]: - here = here[t] - here["__value__"] = payload - # tags - tags = {} - tags["device_id"] = topics[1] - # case of device attribute - device_attributes = ["$homie", "$name", "$state", "$nodes", "$extensions", "$implementation"] - if len(topics) == 3 and topics[2] in device_attributes: - measurement = topics[2] - fields = {"value": payload} - write_point(measurement, tags, fields) - # case of device node property - elif len(topics) == 4: - measurement = topics[3] - tags["node"] = topics[2] - datatype = homie[topics[1]][topics[2]][topics[3]]["$datatype"]["__value__"] - print(datatype, datatype=="float") - if datatype == "float": - fields = {"value": float(payload)} - else: - raise KeyError - write_point(measurement, tags, fields) - - -influx_client = InfluxDBClient('localhost', 8086, 'root', 'root', "homie") -influx_client.create_database("homie") - - -def write_point(measurement, tags, fields): - json = {} - json["measurement"] = measurement - json["tags"] = tags - json["fields"] = fields - try: - influx_client.write_points([json]) - except Exception as e: - print(e) - print(json) - - - -client = mqtt.Client() -client.on_connect = on_connect -client.on_message = on_message - -client.connect("mosquitto.chem.wisc.edu", 1883, 60) - -# Blocking call that processes network traffic, dispatches callbacks and -# handles reconnecting. -# Other loop*() functions are available that give a threaded interface and a -# manual interface. -client.loop_forever() -- cgit v1.2.3