Исправлена ошибка в чтении блоков, добавлен запрос сырых бинарных данных, добавлен параметр seek

.appveyor.yml

@@ -1,26 +0,0 @@
-version: '{build}'
-image: Visual Studio 2015
-platform:
-- x86
-- x64
-environment:
-  global:
-    DISTUTILS_USE_SDK: 1
-    PYTHONWARNINGS: ignore:DEPRECATION
-    MSSdk: 1
-  matrix:
-  - PYTHON: 36
-install:
-- cmd: '"%VS140COMNTOOLS%\..\..\VC\vcvarsall.bat" %PLATFORM%'
-- ps: |
-    if ($env:PLATFORM -eq "x64") { $env:PYTHON = "$env:PYTHON-x64" }
-    $env:PATH = "C:\Python$env:PYTHON\;C:\Python$env:PYTHON\Scripts\;$env:PATH"
-    python -m pip install --disable-pip-version-check --upgrade --no-warn-script-location pip build virtualenv
-build_script:
-- ps: |
-    python -m build -s
-    cd dist
-    python -m pip install --verbose python_example-0.0.1.tar.gz
-    cd ..
-test_script:
-- ps: python tests\test.py

.github/dependabot.yml

@@ -1,16 +0,0 @@
-version: 2
-updates:
-  # Maintain dependencies for GitHub Actions
-  - package-ecosystem: "github-actions"
-    directory: "/"
-    schedule:
-      interval: "daily"
-    ignore:
-      # Official actions have moving tags like v1
-      # that are used, so they don't need updates here
-      - dependency-name: "actions/checkout"
-      - dependency-name: "actions/setup-python"
-      - dependency-name: "actions/cache"
-      - dependency-name: "actions/upload-artifact"
-      - dependency-name: "actions/download-artifact"
-      - dependency-name: "actions/labeler"

.github/workflows/conda.yml

@@ -1,44 +0,0 @@
-name: Conda
-
-on:
-  workflow_dispatch:
-  push:
-    branches:
-      - master
-  pull_request:
-
-jobs:
-  build:
-    strategy:
-      fail-fast: false
-      matrix:
-        platform: [ubuntu-latest, windows-latest, macos-latest]
-        python-version: ["3.6", "3.8"]
-
-    runs-on: ${{ matrix.platform }}
-
-    # The setup-miniconda action needs this to activate miniconda
-    defaults:
-      run:
-        shell: "bash -l {0}"
-
-    steps:
-      - uses: actions/checkout@v2
-
-      - name: Get conda
-        uses: conda-incubator/setup-miniconda@v2.1.1
-        with:
-          python-version: ${{ matrix.python-version }}
-          channels: conda-forge
-
-      - name: Prepare
-        run: conda install conda-build conda-verify
-
-      - name: Build
-        run: conda build conda.recipe
-
-      - name: Install
-        run: conda install -c ${CONDA_PREFIX}/conda-bld/ python_example
-
-      - name: Test
-        run: python tests/test.py

.github/workflows/pip.yml

@@ -1,66 +0,0 @@
-name: Pip
-
-on:
-  workflow_dispatch:
-  pull_request:
-  push:
-    branches:
-      - master
-
-jobs:
-  build:
-    strategy:
-      fail-fast: false
-      matrix:
-        platform: [windows-latest, macos-latest, ubuntu-latest]
-        python-version: ["3.6", "3.10"]
-
-    runs-on: ${{ matrix.platform }}
-
-    steps:
-    - uses: actions/checkout@v2
-
-    - uses: actions/setup-python@v2
-      with:
-        python-version: ${{ matrix.python-version }}
-
-    - name: Add requirements
-      run: python -m pip install --upgrade wheel setuptools
-
-    - name: Build and install
-      run: pip install --verbose .
-
-    - name: Test
-      run: python tests/test.py
-
-  build-mingw64:
-    runs-on: windows-latest
-    defaults:
-      run:
-        shell: msys2 {0}
-    steps:
-    - uses: msys2/setup-msys2@v2
-      with:
-        update: true
-        install: >-
-          mingw-w64-x86_64-gcc
-          mingw-w64-x86_64-python-pip
-          mingw-w64-x86_64-python-wheel
-
-    - uses: actions/checkout@v2
-
-    - name: Install pybind11
-      # This is required because --no-build-isolation disable dependences
-      # installation
-      run: pip install pybind11
-
-    - name: Build and install
-      # --no-build-isolation is required because the vanilla setuptool does not
-      # support Mingw64.See patches here:
-      # https://github.com/msys2/MINGW-packages/tree/master/mingw-w64-python-setuptools
-      # Without those patches build_ext fails with:
-      # error: --plat-name must be one of ('win32', 'win-amd64', 'win-arm32', 'win-arm64')
-      run: pip install --no-build-isolation .
-
-    - name: Test
-      run: python tests/test.py

.github/workflows/wheels.yml

@@ -1,73 +0,0 @@
-name: Wheels
-
-on:
-  workflow_dispatch:
-  pull_request:
-  push:
-    branches:
-     - master
-  release:
-    types:
-      - published
-
-jobs:
-  build_sdist:
-    name: Build SDist
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-
-    - name: Build SDist
-      run: pipx run build --sdist
-
-    - name: Check metadata
-      run: pipx run twine check dist/*
-
-    - uses: actions/upload-artifact@v2
-      with:
-        path: dist/*.tar.gz
-
-
-  build_wheels:
-    name: Wheels on ${{ matrix.os }}
-    runs-on: ${{ matrix.os }}
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [ubuntu-latest, windows-latest, macos-latest]
-
-    steps:
-    - uses: actions/checkout@v2
-
-    - uses: pypa/cibuildwheel@v2.4.0
-      env:
-        CIBW_ARCHS_MACOS: auto universal2
-
-    - name: Verify clean directory
-      run: git diff --exit-code
-      shell: bash
-
-    - name: Upload wheels
-      uses: actions/upload-artifact@v2
-      with:
-        path: wheelhouse/*.whl
-
-
-  upload_all:
-    name: Upload if release
-    needs: [build_wheels, build_sdist]
-    runs-on: ubuntu-latest
-    if: github.event_name == 'release' && github.event.action == 'published'
-
-    steps:
-    - uses: actions/setup-python@v2
-
-    - uses: actions/download-artifact@v2
-      with:
-        name: artifact
-        path: dist
-
-    - uses: pypa/gh-action-pypi-publish@v1.5.0
-      with:
-        user: __token__
-        password: ${{ secrets.pypi_password }}

.pre-commit-config.yaml

@@ -1,66 +0,0 @@
-# To use:
-#
-#     pre-commit run -a
-#
-# Or:
-#
-#     pre-commit install  # (runs every time you commit in git)
-#
-# To update this file:
-#
-#     pre-commit autoupdate
-#
-# See https://github.com/pre-commit/pre-commit
-
-ci:
-  autoupdate_commit_msg: "chore: update pre-commit hooks"
-  autofix_commit_msg: "style: pre-commit fixes"
-
-repos:
-# Standard hooks
-- repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: v4.0.1
-  hooks:
-  - id: check-added-large-files
-  - id: check-case-conflict
-  - id: check-merge-conflict
-  - id: check-symlinks
-  - id: check-yaml
-    exclude: ^conda\.recipe/meta\.yaml$
-  - id: debug-statements
-  - id: end-of-file-fixer
-  - id: mixed-line-ending
-  - id: requirements-txt-fixer
-  - id: trailing-whitespace
-
-# Black, the code formatter, natively supports pre-commit
-- repo: https://github.com/psf/black
-  rev: 21.9b0
-  hooks:
-  - id: black
-    files: ^(docs)
-
-# Sort your imports in a standard form
-- repo: https://github.com/PyCQA/isort
-  rev: 5.9.3
-  hooks:
-  - id: isort
-
-# Upgrade older Python syntax
-- repo: https://github.com/asottile/pyupgrade
-  rev: v2.29.0
-  hooks:
-  - id: pyupgrade
-    args: ["--py36-plus"]
-
-# Changes tabs to spaces
-- repo: https://github.com/Lucas-C/pre-commit-hooks
-  rev: v1.1.10
-  hooks:
-  - id: remove-tabs
-
-# Suggested hook if you add a .clang-format file
-# - repo: https://github.com/pre-commit/mirrors-clang-format
-#  rev: v13.0.0
-#  hooks:
-#  - id: clang-format

.travis.yml

@@ -1,36 +0,0 @@
-language: cpp
-dist: trusty
-matrix:
-  include:
-  - os: linux
-    env: PYTHON=3.8
-  - os: linux
-    env: CONDA=3.7
-before_install:
-- |
-  export CXX=g++-4.8 CC=gcc-4.8
-  if [ -n "$PYTHON" ]; then
-    python -m pip install --user virtualenv
-    virtualenv -p python${PYTHON:0:1} venv
-    source venv/bin/activate
-  elif [ -n "$CONDA" ]; then
-    wget -O miniconda.sh https://repo.continuum.io/miniconda/Miniconda${CONDA:0:1}-latest-Linux-x86_64.sh
-    bash miniconda.sh -b -p $HOME/miniconda
-    export PATH="$HOME/miniconda/bin:$PATH"
-    conda config --set always_yes yes --set changeps1 no
-    conda config --add channels conda-forge
-    conda update -q conda
-    conda install -q conda-build
-    conda create -q -n test-environment python=$CONDA
-    source activate test-environment
-  fi
-install:
-- |
-  if [ -n "$PYTHON" ]; then
-    python -m pip install .
-  elif [ -n "$CONDA" ]; then
-    conda build conda.recipe --python $CONDA
-    conda install --use-local python_example
-  fi
-script:
-- python tests/test.py

setup.py

@@ -6,7 +6,7 @@ from pybind11 import get_cmake_dir
 from pybind11.setup_helpers import Pybind11Extension, build_ext
 from setuptools import setup
 
-__version__ = "0.0.1"
+__version__ = "0.1.3"
 
 ext_modules = [
     Pybind11Extension(
@@ -26,7 +26,6 @@ setup(
     description="Read binary files from lightning detection network based on Boltek-StormPCI",
     long_description="",
     ext_modules=ext_modules,
-    extras_require={"test": "pytest"},
     # Currently, build_ext only provides an optional "highest supported C++
     # level" feature, but in the future it may provide more features.
     cmdclass={"build_ext": build_ext},

src/main.cpp

@@ -3,7 +3,14 @@
 #include <pybind11/stl.h>
 #include <vector>
 #include <fstream>
+#include <iostream>
+#include <ctime>
+#include <cmath>
 #include "stormpci.h"
+#include "string.h"
+
+namespace py = pybind11;
+
 
 #define STRINGIFY(x) #x
 #define MACRO_STRINGIFY(x) STRINGIFY(x)
@@ -18,13 +25,35 @@ typedef struct {
 } BlockBinary;
 
 
+double haversine(double angle){
+    return sin(angle / 2.0)*sin(angle / 2.0);
+}
+
+double inverse_haversine(double value){
+    return 2.0 * asin(sqrt(value));
+}
+
+
+double distance(double lat1, double lon1, double lat2, double lon2)
+{
+    double theta = (lon2 - lon1) * M_PI / 180;
+    double phi1 = lat1 * M_PI / 180;
+    double phi2 = lat2 * M_PI / 180;
+    double curve = inverse_haversine(haversine(phi1 - phi2) +
+                              cos(phi1) * cos(phi2) * haversine(theta));
+    return curve * 6'371'000;
+}
+
+
 class BlockReader
 {
 public:
-    BlockReader(const std::string &fn){
+    BlockReader(const std::string &fn, long int seek_position){
         f = std::ifstream(fn, std::fstream::binary);
+        pos = seek_position;
     }
     BlockBinary& read_block(){
+        f.seekg(pos);
         f.read(reinterpret_cast<char*>(&_single_block), sizeof(BlockBinary));
         if (not f){
             throw;
@@ -34,13 +63,16 @@ public:
         return _single_block;
     }
     std::vector<BlockBinary>& read_all_blocks(){
+        f.seekg(pos);
         bool read_successfully = true;
         while (read_successfully){
             f.read(reinterpret_cast<char*>(&_single_block), sizeof(BlockBinary));
-            if (not f){
+            if (f) {
+                _blocks_vector.push_back(_single_block);
+                pos += sizeof(BlockBinary);
+            } else {
                 read_successfully = false;
             }
-            _blocks_vector.push_back(_single_block);
         }
         return _blocks_vector;
     }
@@ -54,7 +86,6 @@ private:
     std::vector<BlockBinary> _blocks_vector;
 };
 
-namespace py = pybind11;
 
 PYBIND11_MODULE(stormpci_bin_reader, m) {
     m.doc() = R"pbdoc(
@@ -70,7 +101,7 @@ PYBIND11_MODULE(stormpci_bin_reader, m) {
 
 
     py::class_<BlockReader>(m, "BlockReader")
-        .def(py::init<const std::string &>())
+        .def(py::init<const std::string &, long int>(), py::arg("fn"), py::arg("seek")=int(0))
         .def("read_block", &BlockReader::read_block)
         .def("read_all_blocks", &BlockReader::read_all_blocks);
 
@@ -78,6 +109,9 @@ PYBIND11_MODULE(stormpci_bin_reader, m) {
         .def_readonly("strike", &BlockBinary::strike)
         .def_readonly("device_id", &BlockBinary::device_id)
         .def_readonly("experiment_id", &BlockBinary::experiment_id)
+        .def_property_readonly("raw_data", [](const BlockBinary bb) {
+            return py::bytes(reinterpret_cast<const char*>(&bb), sizeof(bb));
+        })
         .def("__repr__",
             [](const BlockBinary &bb) {
                 std::stringstream stream;
@@ -155,6 +189,43 @@ PYBIND11_MODULE(stormpci_bin_reader, m) {
         .def_readonly("EastPol", &StrikeData::EastPol)
         .def_readonly("EFieldPol", &StrikeData::EFieldPol)
         .def_readonly("lts2_data", &StrikeData::lts2_data)
+        .def_property_readonly("_timestamp_ns", [](const StrikeData &strike) {
+
+            std::tm tm {};
+
+            tm.tm_year = (int)strike.lts2_data.year - 1900;
+            tm.tm_mon = (int)strike.lts2_data.month - 1;
+            tm.tm_mday = (int)strike.lts2_data.day;
+            tm.tm_hour = (int)strike.lts2_data.hours;
+            tm.tm_min = (int)strike.lts2_data.minutes;
+            tm.tm_sec = (int)strike.lts2_data.seconds;
+            tm.tm_isdst = 0;
+
+            std::time_t epoch_s  =  std::mktime(&tm);
+            unsigned long long oscillator_ns = ((unsigned long long)strike.lts2_data.TS_time*1'000'000'000)/(unsigned long long)strike.lts2_data.TS_Osc;
+            unsigned long long timestamp_ns = epoch_s*1'000'000'000 + oscillator_ns;
+           
+            return timestamp_ns;
+        })
+        .def_property_readonly("_station_coords", [](const StrikeData &strike) {
+
+            double lat = (double)strike.lts2_data.latitude_mas / 324000000 * 90;
+            double lon = (double)strike.lts2_data.longitude_mas / 648000000 * 180;
+
+            switch(strike.lts2_data.latitude_ns)
+            {
+                case 'N': break;
+                case 'S': lat = -lat; break;
+            }
+            switch(strike.lts2_data.longitude_ew)
+            {
+                case 'E': break;
+                case 'W': lon = -lon; break;
+            }
+            return std::pair(lat, lon);
+        })
+
+
         ;
 
 

tests/test.py

@@ -1,7 +1,18 @@
 from stormpci_bin_reader import BlockReader
 
-br = BlockReader("2022-04-23-00:00:37.bin")
+br = BlockReader("2022-04-23-00:00:37.bin", seek=0)
 blocks = br.read_all_blocks()
 
-for block in blocks:
+# print(len(blocks))
+
+for j, block in enumerate(blocks):
+    if j == 0:
+        # print(len(block.raw_data))
+        with open("test.bin", "wb") as f:
+            f.write(block.raw_data)
     print(block)
+
+# for tests run in bash:
+#   head -c 6344 2022-04-23-00\:00\:37.bin > test2.bin
+# and
+#   diff test2.bin test.bin