From 024acdc7794e521c9e62f741cb15dccad40fc617 Mon Sep 17 00:00:00 2001 From: Allen Hill Date: Sun, 21 Jun 2026 15:23:25 -0700 Subject: [PATCH] Switch to a python based [streaming] packet capture setup Co-Authored-By: Claude Opus 4.8 --- scripts/packet-analysis/Manifest.toml | 212 -------- scripts/packet-analysis/Project.toml | 10 - scripts/packet-analysis/README.md | 33 +- scripts/packet-analysis/dev/PcapTools/LICENSE | 19 - .../dev/PcapTools/Project.toml | 21 - .../packet-analysis/dev/PcapTools/README.md | 5 - .../dev/PcapTools/src/PcapTools.jl | 26 - .../dev/PcapTools/src/buffer_reader.jl | 96 ---- .../dev/PcapTools/src/pcap_header.jl | 46 -- .../dev/PcapTools/src/record.jl | 25 - .../dev/PcapTools/src/record_header.jl | 14 - .../dev/PcapTools/src/splitcap.jl | 102 ---- .../dev/PcapTools/src/stream_reader.jl | 63 --- .../dev/PcapTools/src/stream_writer.jl | 30 -- scripts/packet-analysis/extcap/avclan-pcap.py | 456 ++++++++++++++++++ scripts/packet-analysis/log_to_pcap.py | 156 ++++++ scripts/packet-analysis/src/AVCLANPipe.jl | 95 ---- .../src/convert-sr-annotation-export.jl | 55 --- scripts/packet-analysis/src/pipepackets.jl | 45 -- 19 files changed, 641 insertions(+), 868 deletions(-) delete mode 100644 scripts/packet-analysis/Manifest.toml delete mode 100644 scripts/packet-analysis/Project.toml delete mode 100644 scripts/packet-analysis/dev/PcapTools/LICENSE delete mode 100644 scripts/packet-analysis/dev/PcapTools/Project.toml delete mode 100644 scripts/packet-analysis/dev/PcapTools/README.md delete mode 100644 scripts/packet-analysis/dev/PcapTools/src/PcapTools.jl delete mode 100644 scripts/packet-analysis/dev/PcapTools/src/buffer_reader.jl delete mode 100644 scripts/packet-analysis/dev/PcapTools/src/pcap_header.jl delete mode 100644 scripts/packet-analysis/dev/PcapTools/src/record.jl delete mode 100644 scripts/packet-analysis/dev/PcapTools/src/record_header.jl delete mode 100644 scripts/packet-analysis/dev/PcapTools/src/splitcap.jl delete mode 100644 scripts/packet-analysis/dev/PcapTools/src/stream_reader.jl delete mode 100644 scripts/packet-analysis/dev/PcapTools/src/stream_writer.jl create mode 100755 scripts/packet-analysis/extcap/avclan-pcap.py create mode 100644 scripts/packet-analysis/log_to_pcap.py delete mode 100644 scripts/packet-analysis/src/AVCLANPipe.jl delete mode 100644 scripts/packet-analysis/src/convert-sr-annotation-export.jl delete mode 100644 scripts/packet-analysis/src/pipepackets.jl diff --git a/scripts/packet-analysis/Manifest.toml b/scripts/packet-analysis/Manifest.toml deleted file mode 100644 index 471967f..0000000 --- a/scripts/packet-analysis/Manifest.toml +++ /dev/null @@ -1,212 +0,0 @@ -# This file is machine-generated - editing it directly is not advised - -julia_version = "1.12.0" -manifest_format = "2.0" -project_hash = "eb09ad96f566b90998c9665d6eb0aaba02e61d92" - -[[deps.AVCLANPipe]] -deps = ["Dates", "LibSerialPort", "PcapTools", "UnixTimes"] -path = "." -uuid = "bfacebfd-1bd5-49e7-81bd-5edd8f4074de" -version = "0.1.0" - -[[deps.ArgTools]] -uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" -version = "1.1.2" - -[[deps.Artifacts]] -uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" -version = "1.11.0" - -[[deps.Base64]] -uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" -version = "1.11.0" - -[[deps.Dates]] -deps = ["Printf"] -uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" -version = "1.11.0" - -[[deps.Downloads]] -deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"] -uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6" -version = "1.6.0" - -[[deps.FileWatching]] -uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" -version = "1.11.0" - -[[deps.JLLWrappers]] -deps = ["Artifacts", "Preferences"] -git-tree-sha1 = "0533e564aae234aff59ab625543145446d8b6ec2" -uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210" -version = "1.7.1" - -[[deps.JuliaSyntaxHighlighting]] -deps = ["StyledStrings"] -uuid = "ac6e5ff7-fb65-4e79-a425-ec3bc9c03011" -version = "1.12.0" - -[[deps.LibCURL]] -deps = ["LibCURL_jll", "MozillaCACerts_jll"] -uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21" -version = "0.6.4" - -[[deps.LibCURL_jll]] -deps = ["Artifacts", "LibSSH2_jll", "Libdl", "OpenSSL_jll", "Zlib_jll", "nghttp2_jll"] -uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" -version = "8.11.1+1" - -[[deps.LibGit2]] -deps = ["LibGit2_jll", "NetworkOptions", "Printf", "SHA"] -uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" -version = "1.11.0" - -[[deps.LibGit2_jll]] -deps = ["Artifacts", "LibSSH2_jll", "Libdl", "OpenSSL_jll"] -uuid = "e37daf67-58a4-590a-8e99-b0245dd2ffc5" -version = "1.9.0+0" - -[[deps.LibSSH2_jll]] -deps = ["Artifacts", "Libdl", "OpenSSL_jll"] -uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8" -version = "1.11.3+1" - -[[deps.LibSerialPort]] -deps = ["Libdl", "libserialport_jll"] -git-tree-sha1 = "69403284a6217f6e60e76553032a103893b7d910" -uuid = "a05a14c7-6e3b-5ba9-90a2-45558833e1df" -version = "0.5.2" - -[[deps.Libdl]] -uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" -version = "1.11.0" - -[[deps.Logging]] -uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" -version = "1.11.0" - -[[deps.Markdown]] -deps = ["Base64", "JuliaSyntaxHighlighting", "StyledStrings"] -uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" -version = "1.11.0" - -[[deps.Mmap]] -uuid = "a63ad114-7e13-5084-954f-fe012c677804" -version = "1.11.0" - -[[deps.MozillaCACerts_jll]] -uuid = "14a3606d-f60d-562e-9121-12d972cd8159" -version = "2025.5.20" - -[[deps.NetworkOptions]] -uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" -version = "1.3.0" - -[[deps.OpenSSL_jll]] -deps = ["Artifacts", "Libdl"] -uuid = "458c3c95-2e84-50aa-8efc-19380b2a3a95" -version = "3.5.1+0" - -[[deps.PcapTools]] -deps = ["Dates", "Mmap", "UnixTimes", "UnsafeArrays"] -path = "dev/PcapTools" -uuid = "222fe7e8-3f39-464a-bf97-d9bbb753f246" -version = "1.1.1" - -[[deps.Pkg]] -deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "Random", "SHA", "TOML", "Tar", "UUIDs", "p7zip_jll"] -uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" -version = "1.12.0" - - [deps.Pkg.extensions] - REPLExt = "REPL" - - [deps.Pkg.weakdeps] - REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" - -[[deps.Preferences]] -deps = ["TOML"] -git-tree-sha1 = "0f27480397253da18fe2c12a4ba4eb9eb208bf3d" -uuid = "21216c6a-2e73-6563-6e65-726566657250" -version = "1.5.0" - -[[deps.Printf]] -deps = ["Unicode"] -uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" -version = "1.11.0" - -[[deps.Random]] -deps = ["SHA"] -uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" -version = "1.11.0" - -[[deps.SHA]] -uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" -version = "0.7.0" - -[[deps.StyledStrings]] -uuid = "f489334b-da3d-4c2e-b8f0-e476e12c162b" -version = "1.11.0" - -[[deps.TOML]] -deps = ["Dates"] -uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76" -version = "1.0.3" - -[[deps.Tar]] -deps = ["ArgTools", "SHA"] -uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e" -version = "1.10.0" - -[[deps.UUIDs]] -deps = ["Random", "SHA"] -uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" -version = "1.11.0" - -[[deps.Unicode]] -uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" -version = "1.11.0" - -[[deps.UnixTimes]] -deps = ["Dates"] -git-tree-sha1 = "1add5a01e8b2f45371d96583595ab2bf734a49e3" -uuid = "ab1a18e7-b408-4913-896c-624bb82ed7f4" -version = "1.7.2" - - [deps.UnixTimes.extensions] - UnixTimesMakieExt = ["Makie", "Observables"] - UnixTimesStructTypesExt = "StructTypes" - UnixTimesTimeZonesExt = "TimeZones" - - [deps.UnixTimes.weakdeps] - Makie = "ee78f7c6-11fb-53f2-987a-cfe4a2b5a57a" - Observables = "510215fc-4207-5dde-b226-833fc4488ee2" - StructTypes = "856f2bd8-1eba-4b0a-8007-ebc267875bd4" - TimeZones = "f269a46b-ccf7-5d73-abea-4c690281aa53" - -[[deps.UnsafeArrays]] -git-tree-sha1 = "efcbb6828697ffffc245a9de7300552535abe005" -uuid = "c4a57d5a-5b31-53a6-b365-19f8c011fbd6" -version = "1.0.8" - -[[deps.Zlib_jll]] -deps = ["Libdl"] -uuid = "83775a58-1f1d-513f-b197-d71354ab007a" -version = "1.3.1+2" - -[[deps.libserialport_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "aacc5af8e8ba0b48a6feadac7afd4bca5d4e98d8" -uuid = "220460dc-b50e-5ed0-8176-09b0fd261e90" -version = "0.1.3+0" - -[[deps.nghttp2_jll]] -deps = ["Artifacts", "Libdl"] -uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" -version = "1.64.0+1" - -[[deps.p7zip_jll]] -deps = ["Artifacts", "Libdl"] -uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0" -version = "17.5.0+2" diff --git a/scripts/packet-analysis/Project.toml b/scripts/packet-analysis/Project.toml deleted file mode 100644 index cc8f42f..0000000 --- a/scripts/packet-analysis/Project.toml +++ /dev/null @@ -1,10 +0,0 @@ -name = "AVCLANPipe" -uuid = "bfacebfd-1bd5-49e7-81bd-5edd8f4074de" -authors = ["Allen Hill "] -version = "0.1.0" - -[deps] -Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" -LibSerialPort = "a05a14c7-6e3b-5ba9-90a2-45558833e1df" -PcapTools = "222fe7e8-3f39-464a-bf97-d9bbb753f246" -UnixTimes = "ab1a18e7-b408-4913-896c-624bb82ed7f4" diff --git a/scripts/packet-analysis/README.md b/scripts/packet-analysis/README.md index a3ea514..e2e4f6f 100644 --- a/scripts/packet-analysis/README.md +++ b/scripts/packet-analysis/README.md @@ -5,7 +5,32 @@ - Copy or link `avclan_plugin.lua` to the Wireshark Lua Pluginds folder - Linking is more convenient if modifying/developing the dissector `ln -s $(pwd)/avclan_plugin.lua ~/.local/lib/wireshark/plugins/avclan_plugin.lua` -- Install Julia and needed packages (I recommend using [juliaup](https://github.com/JuliaLang/juliaup) or an official binary from the Julialang website. Binaries from your distribution are typically out of date and/or built incorrectly.) - - From this folder, start Julia with the local project environment using `julia --project=@.` - - Run `] instantiate` to install the necessary Julia packages -- Run `julia src/pipepackets.jl | wireshark -k -i -` to pipe packets logged by the Mockingboard over serial into Wireshark. If the Lua dissector is installed correctly, the packets should be correctly recognized and dissected as IEBUS/AVCLAN packets \ No newline at end of file +- The bridge script ([extcap/avclan-pcap.py](extcap/avclan-pcap.py)) needs Python 3 and, for live capture only, [pyserial](https://pypi.org/project/pyserial/). Install pyserial so the `python3` that Wireshark runs can import it (e.g. `pip install pyserial`, or your distro's `python3-serial`). The `--convert` mode and the Wireshark interface listing work without it. (Alternatively, run the script via [uv](https://docs.astral.sh/uv/) — `uv run --script extcap/avclan-pcap.py …` — which resolves the inline PEP 723 dependency automatically; but the GUI extcap uses plain `python3`, so the install above is what makes capture work there.) + +## Live capture (Wireshark GUI) + +The easiest way to stream packets is as an [extcap](https://www.wireshark.org/docs/man-pages/extcap.html) interface, which appears right in Wireshark's capture list: + +- Make the script executable and link it into Wireshark's **Personal Extcap path** (Help > About Wireshark, Folders tab — alongside the Lua plugin folder above): + + chmod +x extcap/avclan-pcap.py + mkdir ~/.local/lib/wireshark/extcap + ln -s "$(pwd)/extcap/avclan-pcap.py" ~/.local/lib/wireshark/extcap/avclan-pcap.py + +- Restart Wireshark. **AVC-LAN (serial)** now shows up in the interface list. The script autodetects a USB-UART adapter; to override, open the interface's gear/options and set the serial port. Click the shark fin to start. Wireshark owns the process, so stopping the capture (or unplugging the adapter) shuts the bridge down cleanly. + +## Live capture (command line) + +If you prefer the pipe, the same script works as a plain producer: + + ./extcap/avclan-pcap.py --port /dev/ttyUSB0 | wireshark -k -i - + +Omit `--port` (or pass `--port auto`) to autodetect the adapter. + +## Converting a text log to pcap + +The Mockingboard's text-mode REPL output (e.g. [msgdumps/myfile.txt](msgdumps/myfile.txt)) can be turned into a capture file offline: + + ./extcap/avclan-pcap.py --convert msgdumps/myfile.txt -o out.pcap + +Use a `.pcapng` extension to get pcapng instead (requires `editcap`, which ships with Wireshark). With the Lua dissector installed, the result is recognized and dissected as IEBUS/AVCLAN packets. diff --git a/scripts/packet-analysis/dev/PcapTools/LICENSE b/scripts/packet-analysis/dev/PcapTools/LICENSE deleted file mode 100644 index 701f519..0000000 --- a/scripts/packet-analysis/dev/PcapTools/LICENSE +++ /dev/null @@ -1,19 +0,0 @@ -Copyright (c) 2020 Christian Rorvik - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/scripts/packet-analysis/dev/PcapTools/Project.toml b/scripts/packet-analysis/dev/PcapTools/Project.toml deleted file mode 100644 index f819b5d..0000000 --- a/scripts/packet-analysis/dev/PcapTools/Project.toml +++ /dev/null @@ -1,21 +0,0 @@ -name = "PcapTools" -uuid = "222fe7e8-3f39-464a-bf97-d9bbb753f246" -authors = ["Christian Rorvik "] -version = "1.1.1" - -[deps] -Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" -Mmap = "a63ad114-7e13-5084-954f-fe012c677804" -UnixTimes = "ab1a18e7-b408-4913-896c-624bb82ed7f4" -UnsafeArrays = "c4a57d5a-5b31-53a6-b365-19f8c011fbd6" - -[compat] -UnixTimes = "1" -UnsafeArrays = "1" -julia = "1.4" - -[extras] -Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" - -[targets] -test = ["Test"] diff --git a/scripts/packet-analysis/dev/PcapTools/README.md b/scripts/packet-analysis/dev/PcapTools/README.md deleted file mode 100644 index 8a29171..0000000 --- a/scripts/packet-analysis/dev/PcapTools/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# PcapTools - -[![Build Status](https://travis-ci.com/ancapdev/PcapTools.jl.svg?branch=master)](https://travis-ci.com/ancapdev/PcapTools.jl) - -[![Coverage Status](https://coveralls.io/repos/github/ancapdev/PcapTools.jl/badge.svg)](https://coveralls.io/github/ancapdev/PcapTools.jl) diff --git a/scripts/packet-analysis/dev/PcapTools/src/PcapTools.jl b/scripts/packet-analysis/dev/PcapTools/src/PcapTools.jl deleted file mode 100644 index 2299d36..0000000 --- a/scripts/packet-analysis/dev/PcapTools/src/PcapTools.jl +++ /dev/null @@ -1,26 +0,0 @@ -module PcapTools - -using Dates -using Mmap -using UnixTimes -using UnsafeArrays - -export PcapHeader, RecordHeader -export PcapRecord -export PcapReader, PcapStreamReader, PcapBufferReader -export PcapWriter, PcapStreamWriter -export LINKTYPE_NULL, LINKTYPE_ETHERNET -export splitcap - -abstract type PcapReader end -abstract type PcapWriter end - -include("pcap_header.jl") -include("record_header.jl") -include("record.jl") -include("buffer_reader.jl") -include("stream_reader.jl") -include("stream_writer.jl") -include("splitcap.jl") - -end diff --git a/scripts/packet-analysis/dev/PcapTools/src/buffer_reader.jl b/scripts/packet-analysis/dev/PcapTools/src/buffer_reader.jl deleted file mode 100644 index 48fc737..0000000 --- a/scripts/packet-analysis/dev/PcapTools/src/buffer_reader.jl +++ /dev/null @@ -1,96 +0,0 @@ -""" -Reads pcap data from an array of bytes. -""" -mutable struct PcapBufferReader <: PcapReader - data::Vector{UInt8} - raw_header::Vector{UInt8} - header::PcapHeader - offset::Int64 - mark::Int64 - usec_mul::Int64 - bswapped::Bool - - @doc """ - PcapBufferReader(data::Vector{UInt8}) - - Create reader over `data`. Will read and process pcap header, - and yield records through `read(::PcapBufferReader)`. - """ - function PcapBufferReader(data::Vector{UInt8}) - length(data) < sizeof(PcapHeader) && throw(EOFError()) - rh = data[1:sizeof(PcapHeader)] - h = unsafe_load(Ptr{PcapHeader}(pointer(data))) - h, bswapped, nanotime = process_header(h) - new(data, rh, h, sizeof(h), -1, nanotime ? 1 : 1000, bswapped) - end -end - -""" - PcapBufferReader(path::AbstractString) - -Memory map file in `path` and create PcapBufferReader over its content. -""" -function PcapBufferReader(path::AbstractString) - io = open(path) - data = Mmap.mmap(io) - PcapBufferReader(data) -end - -function Base.close(x::PcapBufferReader) - x.data = UInt8[] - x.offset = 0 - nothing -end - -Base.length(x::PcapBufferReader) = length(x.data) -Base.position(x::PcapBufferReader) = x.offset -Base.seek(x::PcapBufferReader, pos) = x.offset = pos; nothing - -function Base.mark(x::PcapBufferReader) - x.mark = x.offset - x.mark -end - -function Base.unmark(x::PcapBufferReader) - if x.mark >= 0 - x.mark = -1 - true - else - false - end -end - -Base.ismarked(x::PcapBufferReader) = x.mark >= 0 - -function Base.reset(x::PcapBufferReader) - !ismarked(x) && error("PcapBufferReader not marked") - x.offset = x.mark - x.mark = -1 - x.offset -end - -Base.eof(x::PcapBufferReader) = (length(x) - x.offset) < sizeof(RecordHeader) - -""" - read(x::PcapBufferReader) -> PcapRecord - -Read one record from pcap data. -Throws `EOFError` if no more data available. -""" -@inline function Base.read(x::PcapBufferReader) - eof(x) && throw(EOFError()) - record_offset = x.offset - p = pointer(x.data) + record_offset - GC.@preserve x begin - h = unsafe_load(Ptr{RecordHeader}(p)) - end - if x.bswapped - h = bswap(h) - end - t1 = (h.ts_sec + x.header.thiszone) * 1_000_000_000 - t2 = Int64(h.ts_usec) * x.usec_mul - t = UnixTime(Dates.UTInstant(Nanosecond(t1 + t2))) - x.offset += sizeof(RecordHeader) + h.incl_len - x.offset > length(x) && error("Insufficient data in pcap record") - PcapRecord(h, t, x.data, record_offset) -end diff --git a/scripts/packet-analysis/dev/PcapTools/src/pcap_header.jl b/scripts/packet-analysis/dev/PcapTools/src/pcap_header.jl deleted file mode 100644 index cedd49e..0000000 --- a/scripts/packet-analysis/dev/PcapTools/src/pcap_header.jl +++ /dev/null @@ -1,46 +0,0 @@ -# NOTE: Not using @enum because it craps out when displaying unknown values -const LINKTYPE_NULL = UInt32(0) -const LINKTYPE_ETHERNET = UInt32(1) - -struct PcapHeader - magic::UInt32 - version_major::UInt16 - version_minor::UInt16 - thiszone::Int32 - sigfigs::UInt32 - snaplen::UInt32 - linktype::UInt32 -end - -function Base.bswap(x::PcapHeader) - PcapHeader( - bswap(x.magic), - bswap(x.version_major), - bswap(x.version_minor), - bswap(x.thiszone), - bswap(x.sigfigs), - bswap(x.snaplen), - bswap(x.linktype)) -end - -function process_header(x::PcapHeader) - if x.magic == 0xa1b2c3d4 - bswapped = false - nanotime = false - elseif x.magic == 0xd4c3b2a1 - bswapped = true - nanotime = false - elseif x.magic == 0xa1b23c4d - bswapped = false - nanotime = true - elseif x.magic == 0x4d3cb2a1 - bswapped = true - nanotime = true - else - throw(ArgumentError("Invalid pcap header")) - end - if bswapped - x = bswap(x) - end - x, bswapped, nanotime -end diff --git a/scripts/packet-analysis/dev/PcapTools/src/record.jl b/scripts/packet-analysis/dev/PcapTools/src/record.jl deleted file mode 100644 index 45d3be1..0000000 --- a/scripts/packet-analysis/dev/PcapTools/src/record.jl +++ /dev/null @@ -1,25 +0,0 @@ -""" -Record of pcap data -""" -struct PcapRecord - header::RecordHeader - timestamp::UnixTime - underlying_data::Vector{UInt8} - record_offset::Int -end - -@inline function record_field_(x::PcapRecord, ::Val{:data}) - offset = getfield(x, :record_offset) + sizeof(RecordHeader) - len = Int(getfield(x, :header).incl_len) - UnsafeArray{UInt8, 1}(pointer(getfield(x, :underlying_data)) + offset, (len,)) -end - -@inline function record_field_(x::PcapRecord, ::Val{:raw}) - offset = getfield(x, :record_offset) - len = sizeof(RecordHeader) + getfield(x, :header).incl_len - UnsafeArray{UInt8, 1}(pointer(getfield(x, :underlying_data)) + offset, (len,)) -end - -@inline record_field_(x::PcapRecord, ::Val{f}) where {f} = getfield(x, f) - -@inline Base.getproperty(x::PcapRecord, f::Symbol) = record_field_(x, Val(f)) diff --git a/scripts/packet-analysis/dev/PcapTools/src/record_header.jl b/scripts/packet-analysis/dev/PcapTools/src/record_header.jl deleted file mode 100644 index 9aabce9..0000000 --- a/scripts/packet-analysis/dev/PcapTools/src/record_header.jl +++ /dev/null @@ -1,14 +0,0 @@ -struct RecordHeader - ts_sec::UInt32 - ts_usec::UInt32 - incl_len::UInt32 - orig_len::UInt32 -end - -function Base.bswap(x::RecordHeader) - RecordHeader( - bswap(x.ts_sec), - bswap(x.ts_usec), - bswap(x.incl_len), - bswap(x.orig_len)) -end diff --git a/scripts/packet-analysis/dev/PcapTools/src/splitcap.jl b/scripts/packet-analysis/dev/PcapTools/src/splitcap.jl deleted file mode 100644 index 68c4041..0000000 --- a/scripts/packet-analysis/dev/PcapTools/src/splitcap.jl +++ /dev/null @@ -1,102 +0,0 @@ -strip_nothing_(::Type{Union{Nothing, T}}) where T = T -strip_nothing_(::Type{T}) where T = T - -progress_noop_(n) = nothing - -mutable struct SplitCapOutput{S} - work_buffer::Vector{UInt8} - complete_buffers::Channel{Vector{UInt8}} - stream::S -end - -# Since Julia (as of 1.5) doesn't support task migration, -# continue with new task after each buffer write, to rebalance across threads -function write_one_and_continue_(output::SplitCapOutput, free_buffers::Channel, pending::Threads.Atomic{Int}) - i = iterate(output.complete_buffers) - if i === nothing - Threads.atomic_sub!(pending, 1) - return nothing - end - b, _ = i - write(output.stream, b) - empty!(b) - put!(free_buffers, b) - Threads.@spawn write_one_and_continue_($output, $free_buffers, $pending) -end - -function splitcap( - ::Type{KeyType}, - ::Type{StreamType}, - reader::PcapReader, - record2key, - key2stream, - progress_callback = progress_noop_; - own_streams::Bool = true -) where {KeyType, StreamType} - buffer_size = 1024 * 1024 * 2 - max_pending_buffers = 4 - outputs = Dict{KeyType, SplitCapOutput{StreamType}}() - free_buffers = Channel{Vector{UInt8}}(Inf) - n = 0 - pending = Threads.Atomic{Int}(0) - try - while !eof(reader) - record = read(reader) - dst = record2key(record) - if dst isa KeyType - output = get!(outputs, dst) do - stream = key2stream(dst) - buffer = sizehint!(UInt8[], buffer_size + 1500) - own_streams && append!(buffer, reader.raw_header) - output = SplitCapOutput{StreamType}( - buffer, - Channel{Vector{UInt8}}(max_pending_buffers), - stream) - Threads.atomic_add!(pending, 1) - Threads.@spawn write_one_and_continue_($output, $free_buffers, $pending) - output - end - append!(output.work_buffer, record.raw) - if length(output.work_buffer) >= buffer_size - put!(output.complete_buffers, output.work_buffer) - if isready(free_buffers) - output.work_buffer = take!(free_buffers) - else - output.work_buffer = sizehint!(UInt8[], buffer_size + 1500) - end - end - end - n += 1 - progress_callback(n) - GC.safepoint() - end - for output in values(outputs) - if !isempty(output.work_buffer) - put!(output.complete_buffers, output.work_buffer) - end - close(output.complete_buffers) - end - while pending[] != 0 - sleep(0.1) - end - finally - if own_streams - for output in values(outputs) - close(output.stream) - end - end - end - nothing -end - -function splitcap( - reader::PcapReader, - record2key, - key2stream, - progress_callback = progress_noop_; - kwargs... -) - KeyType = strip_nothing_(Core.Compiler.return_type(record2key, Tuple{PcapRecord})) - StreamType = Core.Compiler.return_type(key2stream, Tuple{KeyType}) - splitcap(KeyType, StreamType, reader, record2key, key2stream, progress_callback; kwargs...) -end diff --git a/scripts/packet-analysis/dev/PcapTools/src/stream_reader.jl b/scripts/packet-analysis/dev/PcapTools/src/stream_reader.jl deleted file mode 100644 index ef9089d..0000000 --- a/scripts/packet-analysis/dev/PcapTools/src/stream_reader.jl +++ /dev/null @@ -1,63 +0,0 @@ -""" -Reads pcap data from a stream. -""" -mutable struct PcapStreamReader{Src <: IO} <: PcapReader - src::Src - raw_header::Vector{UInt8} - header::PcapHeader - usec_mul::Int64 - bswapped::Bool - record_buffer::Vector{UInt8} - - @doc """ - PcapStreamReader(src::IO) - - Create reader over `src`. Will read and process pcap header, - and yield records through `read(::PcapStreamReader)`. - """ - function PcapStreamReader(src::Src) where {Src <: IO} - raw_header = read(src, sizeof(PcapHeader)) - length(raw_header) != sizeof(PcapHeader) && throw(EOFError()) - h = GC.@preserve raw_header unsafe_load(Ptr{PcapHeader}(pointer(raw_header))) - header, bswapped, nanotime = process_header(h) - new{Src}(src, raw_header, header, nanotime ? 1 : 1000, bswapped, zeros(UInt8, 9000 + sizeof(RecordHeader))) - end -end - -""" - PcapStreamReader(path) - -Open file at `path` and create PcapStreamReader over its content. -""" -PcapStreamReader(path::AbstractString) = PcapStreamReader(open(path)) - -Base.close(x::PcapStreamReader) = close(x.src) -Base.position(x::PcapStreamReader) = position(x.src) -Base.seek(x::PcapStreamReader, pos) = seek(x.src, pos) -Base.mark(x::PcapStreamReader) = mark(x.src) -Base.unmark(x::PcapStreamReader) = unmark(x.src) -Base.ismarked(x::PcapStreamReader) = ismarked(x.src) -Base.reset(x::PcapStreamReader) = reset(x.src) -Base.eof(x::PcapStreamReader) = eof(x.src) - -""" - read(x::PcapStreamReader) -> PcapRecord - -Read one record from pcap data. Record is valid until next read(). -Throws `EOFError` if no more data available. -""" -function Base.read(x::PcapStreamReader) - p = pointer(x.record_buffer) - GC.@preserve x begin - unsafe_read(x.src, p, sizeof(RecordHeader)) - h = unsafe_load(Ptr{RecordHeader}(p)) - if x.bswapped - h = bswap(h) - end - unsafe_read(x.src, p + sizeof(RecordHeader), h.incl_len) - end - t1 = (h.ts_sec + x.header.thiszone) * 1_000_000_000 - t2 = Int64(h.ts_usec) * x.usec_mul - t = UnixTime(Dates.UTInstant(Nanosecond(t1 + t2))) - PcapRecord(h, t, x.record_buffer, 0) -end diff --git a/scripts/packet-analysis/dev/PcapTools/src/stream_writer.jl b/scripts/packet-analysis/dev/PcapTools/src/stream_writer.jl deleted file mode 100644 index 3c2b6c7..0000000 --- a/scripts/packet-analysis/dev/PcapTools/src/stream_writer.jl +++ /dev/null @@ -1,30 +0,0 @@ -struct PcapStreamWriter{Dst <: IO} <: PcapWriter - dst::Dst - - function PcapStreamWriter{Dst}(dst::Dst; thiszone = 0, snaplen = 65535, linktype = LINKTYPE_ETHERNET) where {Dst <: IO} - h = PcapHeader( - 0xa1b23c4d, - 0x0002, - 0x0004, - thiszone, - 0, - snaplen, - linktype) - write(dst, reinterpret(UInt8, [h])) - new(dst) - end -end - -PcapStreamWriter(io::IO; kwargs...) = PcapStreamWriter{typeof(io)}(io; kwargs...) -PcapStreamWriter(path::AbstractString; kwargs...) = PcapStreamWriter(open(path, "w"); kwargs...) - -Base.close(x::PcapStreamWriter) = close(x.dst) - -function Base.write(x::PcapStreamWriter, timestamp::UnixTime, data) - sec, nsec = fldmod(Dates.value(timestamp), 1_000_000_000) - data_length = length(data) - h = RecordHeader(sec, nsec, data_length, data_length) - write(x.dst, reinterpret(UInt8, [h])) - write(x.dst, collect(data)) - nothing -end diff --git a/scripts/packet-analysis/extcap/avclan-pcap.py b/scripts/packet-analysis/extcap/avclan-pcap.py new file mode 100755 index 0000000..bc0bb14 --- /dev/null +++ b/scripts/packet-analysis/extcap/avclan-pcap.py @@ -0,0 +1,456 @@ +#!/usr/bin/env python3 +# /// script +# requires-python = ">=3.9" +# dependencies = ["pyserial"] +# /// +"""AVC-LAN Mockingboard -> pcap bridge for Wireshark. + +Three modes, one script: + + * extcap - invoked by Wireshark; appears as the "AVC-LAN (serial)" capture + interface. Wireshark owns the process lifecycle, so stopping the + capture (or unplugging the adapter) tears everything down cleanly. + * pipe - run by hand: `./avclan-pcap.py --port /dev/ttyUSB0 | wireshark -k -i -` + * convert - turn a text-mode serial log into a pcap file offline: + `./avclan-pcap.py --convert msgdumps/myfile.txt -o out.pcap` + +The firmware emits frames over serial at 1.2 Mbaud, 8N1, in one of two formats +(selected at the REPL). This script understands both and emits pcap with +linktype 162 (USER15), matching the avclan_plugin.lua dissector. + +Discovery (the extcap query phases), `--convert`, and argument handling run on +the standard library alone, so the script keeps a plain `python3` shebang and +Wireshark can always exec it -- Wireshark launches extcaps with a minimal PATH, +which is why a `uv` shebang here would make the interface vanish from the GUI. +The one dependency, `pyserial`, is only needed to actually read the serial port: +capture fails with a clear message if it is missing. Install it normally (so the +`python3` invocation Wireshark uses can find it), or run the script via +`uv run --script` to have uv resolve the inline dependency above. +""" + +import argparse +import os +import re +import shutil +import signal +import struct +import subprocess +import sys +import tempfile +import time +from datetime import datetime, timedelta + +# --------------------------------------------------------------------------- +# pcap on-the-wire format. Mirrors the old Julia PcapTools output exactly: +# nanosecond magic, snaplen 64, linktype 162 (DLT_USER15 / wtap.USER15). +# --------------------------------------------------------------------------- +PCAP_MAGIC_NS = 0xA1B23C4D +SNAPLEN = 64 +LINKTYPE_USER15 = 162 + +PCAP_GLOBAL_HEADER = struct.pack( + " bad framing + +BAUD = 1_200_000 + +# USB-UART bridge vendor IDs worth preferring during autodetect. +PREFERRED_VIDS = { + 0x0403, # FTDI + 0x10C4, # Silicon Labs CP210x + 0x1A86, # WCH CH340/CH341 + 0x067B, # Prolific PL2303 +} + + +# --------------------------------------------------------------------------- +# pcap writing +# --------------------------------------------------------------------------- +def write_global_header(out): + out.write(PCAP_GLOBAL_HEADER) + out.flush() + + +def write_record(out, ts_ns, payload): + sec, nsec = divmod(ts_ns, 1_000_000_000) + out.write(struct.pack(" MAX_DATA_LEN: + pos = dle + 1 # spurious DLE; resync past it + continue + etb = dle + 8 + length # index where ETB must sit + if len(buf) < etb + 1: + del buf[:dle] + return frames + if buf[etb] != ETB: + pos = dle + 1 # framing mismatch; this DLE was not a real start + continue + frames.append(bytes(buf[dle + 1:etb])) # 7 header bytes + data + pos = etb + 1 + while pos < len(buf) and buf[pos] in (0x0D, 0x0A): + pos += 1 # swallow the trailing \r\n + del buf[:pos] + pos = 0 + + +# --------------------------------------------------------------------------- +# Serial helpers (lazy pyserial import) +# --------------------------------------------------------------------------- +def detect_port(): + """Best-guess serial device, or None. Prefers known USB-UART bridges. + + Uses pyserial's enumeration when available, but falls back to a stdlib + glob so the extcap config dialog still offers a sensible default before + pyserial has been pulled in. + """ + try: + from serial.tools import list_ports + except ImportError: + import glob + cands = sorted(glob.glob("/dev/ttyUSB*")) + sorted(glob.glob("/dev/ttyACM*")) + return cands[0] if cands else None + + ports = list(list_ports.comports()) + if not ports: + return None + + def score(p): + s = 0 + if p.vid in PREFERRED_VIDS: + s += 10 + dev = p.device or "" + if "ttyUSB" in dev: + s += 3 + elif "ttyACM" in dev: + s += 2 + return s + + ports.sort(key=score, reverse=True) + return ports[0].device + + +def ensure_pyserial(): + """Fail with a clear message if pyserial (capture-only) is unavailable.""" + try: + import serial # noqa: F401 + except ImportError: + sys.exit("error: capture requires pyserial, which is not installed.\n" + "Install it (e.g. `pip install pyserial`), or run this script via " + "`uv run --script` to pull in the inline dependency.") + + +def resolve_port(requested): + """Map a requested port (possibly None/"auto") to a concrete device.""" + if requested and requested != "auto": + return requested + port = detect_port() + if not port: + sys.exit("error: no serial port given and none could be autodetected " + "(pass --port /dev/ttyUSBx)") + return port + + +def open_serial(port): + import serial # lazy: only the serial modes need it + + # Flow control fully off: the binary payload can legitimately contain + # 0x11 (XON) / 0x13 (XOFF), which a flow-controlled link would eat. + return serial.Serial( + port, + BAUD, + timeout=0.2, # short, so signal flags get checked promptly + xonxoff=False, + rtscts=False, + dsrdtr=False, + ) + + +# --------------------------------------------------------------------------- +# Capture (extcap + pipe) +# --------------------------------------------------------------------------- +_stop = False + + +def _request_stop(signum, frame): + global _stop + _stop = True + + +def run_capture(out, port): + """Stream frames from the serial port to `out` until told to stop.""" + ensure_pyserial() # re-execs under uv if pyserial is missing + import serial # for SerialException + + port = resolve_port(port) + try: + ser = open_serial(port) + except serial.SerialException as e: + sys.exit("error: could not open %s: %s" % (port, e)) + + # Put the firmware into binary-frame mode (REPL 'X'). + try: + ser.write(b"X") + except serial.SerialException: + pass + + buf = bytearray() + try: + write_global_header(out) + while not _stop: + try: + chunk = ser.read(ser.in_waiting or 1) + except serial.SerialException: + break # adapter went away -> stop cleanly + if not chunk: + continue + buf.extend(chunk) + for payload in extract_frames(buf): + write_record(out, time.time_ns(), payload) + except (BrokenPipeError, OSError): + pass # Wireshark closed the fifo/pipe -> we're done + finally: + ser.close() + + +# --------------------------------------------------------------------------- +# Convert (offline text log -> pcap), replacing Julia avclan_text_to_pcap +# --------------------------------------------------------------------------- +TIME_RE = re.compile(r"^(\d{1,2}):(\d{2}):(\d{2})\.(\d+)$") + + +def parse_text_line(line, base_date): + """Parse one text-mode log line. + + Returns (ts_ns_or_None, payload) or None if the line is not a frame. + Line format (whitespace separated): + [HH:MM:SS.mmm] is_unicast controller peripheral control length data... + with the numeric fields in C-style notation (0x.. or plain decimal). + """ + tokens = line.split() + if len(tokens) < 5: + return None + + ts_ns = None + m = TIME_RE.match(tokens[0]) + if m: + h, mn, s, frac = m.groups() + micros = int(frac.ljust(6, "0")[:6]) + try: + dt = datetime(base_date.year, base_date.month, base_date.day, + int(h), int(mn), int(s)) + except ValueError: + return None + ts_ns = int(dt.timestamp()) * 1_000_000_000 + micros * 1000 + tokens = tokens[1:] + + if len(tokens) < 5: + return None + try: + is_unicast = int(tokens[0], 0) + controller = int(tokens[1], 0) + peripheral = int(tokens[2], 0) + control = int(tokens[3], 0) + length = int(tokens[4], 0) + except ValueError: + return None + + data_tokens = tokens[5:] + if length > MAX_DATA_LEN or len(data_tokens) != length: + return None + try: + data = bytes(int(d, 0) & 0xFF for d in data_tokens) + except ValueError: + return None + + payload = bytes([ + is_unicast & 0xFF, + (controller >> 8) & 0xFF, controller & 0xFF, + (peripheral >> 8) & 0xFF, peripheral & 0xFF, + control & 0xFF, + length & 0xFF, + ]) + data + return ts_ns, payload + + +def run_convert(infile, output): + # Date comes from the log's mtime (the lines only carry a wall-clock time); + # falls back to today if that is unavailable. + try: + base_date = datetime.fromtimestamp(os.path.getmtime(infile)) + except OSError: + base_date = datetime.now() + + want_pcapng = bool(output) and output.endswith(".pcapng") + tmp_path = None + if want_pcapng: + fd, tmp_path = tempfile.mkstemp(suffix=".pcap") + out = os.fdopen(fd, "wb") + close_out = True + elif output: + out = open(output, "wb") + close_out = True + else: + out = sys.stdout.buffer + close_out = False + + last_ts = None + count = 0 + try: + write_global_header(out) + with open(infile, "r") as f: + for line in f: + parsed = parse_text_line(line, base_date) + if parsed is None: + continue + ts_ns, payload = parsed + if ts_ns is None: + # No timestamp on this line; nudge past the previous one so + # ordering is preserved (mirrors the old Julia behaviour). + base = last_ts if last_ts is not None else time.time_ns() + ts_ns = base + 1000 + write_record(out, ts_ns, payload) + last_ts = ts_ns + count += 1 + finally: + if close_out: + out.close() + + if want_pcapng: + editcap = shutil.which("editcap") + if editcap: + subprocess.run([editcap, "-F", "pcapng", tmp_path, output], check=True) + os.unlink(tmp_path) + else: + fallback = output[: -len(".pcapng")] + ".pcap" + shutil.move(tmp_path, fallback) + print("warning: editcap not found; wrote pcap to %s instead of %s" + % (fallback, output), file=sys.stderr) + output = fallback + + if output: + print("wrote %d frames to %s" % (count, output), file=sys.stderr) + + +# --------------------------------------------------------------------------- +# extcap argument phases +# --------------------------------------------------------------------------- +def extcap_interfaces(): + print("extcap {version=1.0}{help=https://github.com/}" + "{display=AVC-LAN Mockingboard}") + print("interface {value=avclan}{display=AVC-LAN (serial)}") + + +def extcap_dlts(): + print("dlt {number=%d}{name=USER15}{display=AVC-LAN / IEBus}" % LINKTYPE_USER15) + + +def extcap_config(): + default = detect_port() or "auto" + print('arg {number=0}{call=--port}{display=Serial port}{type=string}' + '{tooltip=Serial device path, or "auto" to autodetect}' + '{default=%s}' % default) + + +# --------------------------------------------------------------------------- +# main +# --------------------------------------------------------------------------- +def main(): + parser = argparse.ArgumentParser(add_help=True) + # extcap discovery/lifecycle flags (Wireshark-driven) + parser.add_argument("--extcap-interfaces", action="store_true") + parser.add_argument("--extcap-dlts", action="store_true") + parser.add_argument("--extcap-config", action="store_true") + parser.add_argument("--extcap-interface") + parser.add_argument("--extcap-version") + parser.add_argument("--capture", action="store_true") + parser.add_argument("--fifo") + parser.add_argument("--extcap-control-in") + parser.add_argument("--extcap-control-out") + # shared / CLI + parser.add_argument("--port", help='serial device, or "auto"') + parser.add_argument("--convert", metavar="LOGFILE", + help="convert a text-mode serial log to pcap") + parser.add_argument("-o", "--output", + help="output file for --convert (.pcap or .pcapng); " + "stdout if omitted") + args, _ = parser.parse_known_args() + + if args.extcap_interfaces: + extcap_interfaces() + return + if args.extcap_dlts: + extcap_dlts() + return + if args.extcap_config: + extcap_config() + return + if args.convert: + run_convert(args.convert, args.output) + return + + signal.signal(signal.SIGINT, _request_stop) + signal.signal(signal.SIGTERM, _request_stop) + + if args.capture: + if not args.fifo: + sys.exit("error: --capture requires --fifo") + with open(args.fifo, "wb") as fifo: + run_capture(fifo, args.port) + return + + # No mode flags: behave as a plain pipe into `wireshark -k -i -`. + run_capture(sys.stdout.buffer, args.port) + + +if __name__ == "__main__": + try: + main() + except BrokenPipeError: + # Reader (Wireshark) went away; exit quietly without a traceback. + try: + sys.stdout.close() + except Exception: + pass + os._exit(0) diff --git a/scripts/packet-analysis/log_to_pcap.py b/scripts/packet-analysis/log_to_pcap.py new file mode 100644 index 0000000..576cd2d --- /dev/null +++ b/scripts/packet-analysis/log_to_pcap.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python3 +"""One-off: convert a bracketed-timestamp AVC-LAN `log` dump into a pcap. + +The capture in `msgdumps/log` is a foreign text format (not the firmware's): + + [ 08:50:12.793 ] 1901F1040025C0EF + +Each hex blob is a tight nibble packing of an AVC-LAN frame, with no spaces and +no leading is_unicast byte (unlike the firmware text format that +extcap/avclan-pcap.py's --convert handles): + + controller : 3 hex nibbles (12-bit address) + peripheral : 3 hex nibbles (12-bit address) + length : 2 hex nibbles (1 byte: data byte count) + data : length bytes (2*length hex nibbles) + +There is no control field in this format; the dissector wants one, so we +synthesize the standard AVC-LAN value (0x0F). + +It emits pcap with linktype 162 (DLT_USER15), the payload layout the +avclan_plugin.lua dissector expects -- byte-for-byte compatible with +avclan-pcap.py's output: + + [is_unicast, ctrl_hi, ctrl_lo, periph_hi, periph_lo, control, length, data...] +""" + +import argparse +import os +import re +import struct +import sys +from datetime import datetime, timedelta + +# pcap on-the-wire format -- mirrors avclan-pcap.py exactly: +# nanosecond magic, snaplen 64, linktype 162 (DLT_USER15 / wtap.USER15). +PCAP_MAGIC_NS = 0xA1B23C4D +SNAPLEN = 64 +LINKTYPE_USER15 = 162 + +PCAP_GLOBAL_HEADER = struct.pack( + " (ts_ns, payload) or None if not a frame.""" + m = LINE_RE.match(line) + if not m: + return None + h, mn, s, frac, hx = m.groups() + + # Header is controller(3) + peripheral(3) + length(2) = 8 nibbles, no control. + if len(hx) < 8: + return None + length = int(hx[6:8], 16) + if len(hx) != 8 + 2 * length: + return None + + controller = int(hx[0:3], 16) + peripheral = int(hx[3:6], 16) + control = STANDARD_CONTROL # not present in the log; synthesize the standard + is_unicast = 0 if peripheral in BROADCAST_ADDRS else 1 + data = bytes.fromhex(hx[8:]) + + payload = bytes([ + is_unicast, + (controller >> 8) & 0xFF, controller & 0xFF, + (peripheral >> 8) & 0xFF, peripheral & 0xFF, + control & 0xFF, + length & 0xFF, + ]) + data + + micros = int(frac.ljust(6, "0")[:6]) + dt = datetime(base_date.year, base_date.month, base_date.day, + int(h), int(mn), int(s)) + return dt, micros, payload + + +def main(): + here = os.path.dirname(os.path.abspath(__file__)) + parser = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("infile", nargs="?", + default=os.path.join(here, "msgdumps", "log"), + help="input log file (default: msgdumps/log)") + parser.add_argument("-o", "--output", + help="output pcap (default: .pcap)") + args = parser.parse_args() + + output = args.output or (args.infile + ".pcap") + + # Lines carry only wall-clock time; take the date from the log's mtime. + try: + base_date = datetime.fromtimestamp(os.path.getmtime(args.infile)) + except OSError: + base_date = datetime.now() + + count = 0 + skipped = 0 + prev_tod = None # previous (h,m,s,micros) for midnight-rollover detection + day_offset = timedelta(0) + + with open(args.infile, "r") as f, open(output, "wb") as out: + write_global_header(out) + for lineno, line in enumerate(f, 1): + if not line.strip(): + continue + parsed = parse_line(line, base_date) + if parsed is None: + skipped += 1 + print("skip: malformed line %d: %r" % (lineno, line.rstrip()), + file=sys.stderr) + continue + dt, micros, payload = parsed + tod = (dt.hour, dt.minute, dt.second, micros) + if prev_tod is not None and tod < prev_tod: + day_offset += timedelta(days=1) # wall clock wrapped past midnight + prev_tod = tod + dt = dt + day_offset + ts_ns = int(dt.timestamp()) * 1_000_000_000 + micros * 1000 + write_record(out, ts_ns, payload) + count += 1 + + print("wrote %d frames to %s%s" % ( + count, output, + " (%d skipped)" % skipped if skipped else ""), file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/scripts/packet-analysis/src/AVCLANPipe.jl b/scripts/packet-analysis/src/AVCLANPipe.jl deleted file mode 100644 index 5c2ea6b..0000000 --- a/scripts/packet-analysis/src/AVCLANPipe.jl +++ /dev/null @@ -1,95 +0,0 @@ -module AVCLANPipe - -using PcapTools, Dates, UnixTimes - -export AVCLANframe, avclan_text_to_pcap, tobytes - -mutable struct AVCLANframe - broadcast::Bool - controller_addr::UInt16 - peripheral_addr::UInt16 - control::UInt8 - length::UInt8 - data::NTuple{32,UInt8} -end - -AVCLANframe() = AVCLANframe(false, 0x0000, 0x0000, 0xf, 0x0, ntuple(x -> 0x0, 32)) - -function Base.tryparse(::Type{AVCLANframe}, str::String) - vals = split(str) - - length(vals) < 6 && return nothing - - local t - try - t = Time(first(vals), "HH:MM:SS.s") - catch e - end - if !(@isdefined(t)) - t = nothing - else - popfirst!(vals) - end - - broadcast = tryparse(Bool, vals[1]) - isnothing(broadcast) && return nothing - - controller_addr = tryparse(UInt16, vals[2]) - isnothing(controller_addr) && return nothing - - peripheral_addr = tryparse(UInt16, vals[3]) - isnothing(peripheral_addr) && return nothing - - control = tryparse(UInt8, vals[4]) - isnothing(control) && return nothing - - len = tryparse(UInt8, vals[5]) - isnothing(len) && return nothing - - if (length(vals) - 5) != len || len > 32 - return nothing - end - _data = tryparse.(UInt8, vals[6:end]) - data = ntuple(i -> checkindex(Bool, axes(_data, 1), i) ? _data[i] : 0x0, 32) - - return t, AVCLANframe(broadcast, controller_addr, peripheral_addr, control, len, data) -end - -function tobytes(frame::AVCLANframe) - data = Vector{UInt8}(undef, 0) - push!(data, frame.broadcast) - append!(data, reverse(reinterpret(reshape, UInt8, [frame.controller_addr])), - reverse(reinterpret(reshape, UInt8, [frame.peripheral_addr])), - reinterpret(reshape, UInt8, [frame.control]), - reinterpret(reshape, UInt8, [frame.length]), - frame.data[1:frame.length]) - - return data -end - -function avclan_text_to_pcap(textlog::String, pcap_fn::String) - pcapstream = PcapStreamWriter(pcap_fn; snaplen=64, linktype=162) - t = UnixTime(now()) - - for line in eachline(textlog) - tstamp_frame = tryparse(AVCLANframe, line) - if !isnothing(tstamp_frame) - tstamp, frame = tstamp_frame - else - continue - end - if !isnothing(tstamp) - t = UnixTime(DateTime(today(), tstamp)) - else - t += Microsecond(rand(1:15)) - end - if !isnothing(frame) - write(pcapstream, t, tobytes(frame)) - end - end - close(pcapstream) - - return pcap_fn -end - -end # module AVCLANPipe diff --git a/scripts/packet-analysis/src/convert-sr-annotation-export.jl b/scripts/packet-analysis/src/convert-sr-annotation-export.jl deleted file mode 100644 index 71dd4e8..0000000 --- a/scripts/packet-analysis/src/convert-sr-annotation-export.jl +++ /dev/null @@ -1,55 +0,0 @@ -using AVCLANPipe, PcapTools, Dates, UnixTimes - -file = ARGS[1] -outfile = ARGS[2] - -function convert_fields_to_pcap(file, outfile) - - io = open(file, "r") - outio = PcapStreamWriter(outfile; snaplen=64, linktype = 162) - - packet = AVCLANframe() - packetcomplete = false - local t::UnixTime - i = 1 - - for line in readlines(io) - m = match(r"(\d+)-(\d+) IEBus: Raw Fields: (?\w+)(:? ((?0x[0-9a-f]{2,3})|WRITE_DATA|Length: (?\d+)))?", - line) - - if m[:field] == "Broadcast" - packet.broadcast = true - t = UnixTime(1970,1,1,0,0,0,0, something(tryparse(Int, m[1]), 0)) - elseif m[:field] == "Unicast" - packet.broadcast = false - t = UnixTime(1970,1,1,0,0,0,0, something(tryparse(Int, m[1]), 0)) - elseif m[:field] == "Master" - packet.controller_addr = tryparse(UInt16, m[:hex]) - elseif m[:field] == "Slave" - packet.peripheral_addr = tryparse(UInt16, m[:hex]) - elseif m[:field] == "Data" - if !isnothing(m[:length]) - packet.length = convert(UInt8, tryparse(Int, m[:length])) - else - data = collect(packet.data) - data[i] = tryparse(UInt8, m[:hex]) - packet.data = ntuple(n -> data[n], 32) - i += 1 - end - end - - if i > packet.length - packetcomplete = true - end - - if packetcomplete - packetcomplete = false - i = 1 - write(outio, t, tobytes(packet)) - end - end - - close(io) - close(outio) - -end diff --git a/scripts/packet-analysis/src/pipepackets.jl b/scripts/packet-analysis/src/pipepackets.jl deleted file mode 100644 index d76e6e3..0000000 --- a/scripts/packet-analysis/src/pipepackets.jl +++ /dev/null @@ -1,45 +0,0 @@ -include("AVCLANPipe.jl") - -using .AVCLANPipe, PcapTools, Dates, UnixTimes, LibSerialPort - -serial_port="/dev/ttyUSB1" -baud=1200000 - -pcapstream = PcapStreamWriter(stdout; snaplen=64, linktype=162) -serial = LibSerialPort.open(serial_port, baud) -set_flow_control(serial) # Disable flow-control (stops it from eating raw byte 0x11) - -write(serial, 'X') -buf = IOBuffer() - -function quit(serialio=serial) - close(serialio) - exit() -end - -while true - iswritable(stdout) || isopen(stdout) || quit() - if bytesavailable(serial) > 0 - t = UnixTime(now()) - line = readline(serial) - # allthebytes = Vector{UInt8}(undef, bytesavailable(serial)) - # readbytes!(serial, allthebytes) - # write(buf, allthebytes) - # l = findfirst(==('\n'), buf.data) - # if !isnothing(l) - # println(stderr, "fucku") - # bytes = Vector{UInt8}(undef, l) - # seekstart(buf) - # readbytes!(buf, bytes) - # seekend(buf) - bytes = Vector{UInt8}(line) - if bytes[1] == 0x10 && bytes[end] == 0x17 - write(pcapstream, t, bytes[2:end-1]) - else - @error line - end - # end - # println(stderr, "wtf") - end - yield() -end