elio/datawriter
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Umsetzung des Delivery Imports

Browse Source
This commit is contained in:
Marco Kittel 2025-06-29 20:37:45 +02:00
parent 4c140eb7d2
commit a522835d7a
5 changed files with 86 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
erzeugeTestdaten.py
View File

@ -7,7 +7,7 @@ def generate_product_id():
number = random.randint(1000, 9999)
return f"{letter}{number}"
def create_csv_file(filename="products.csv", entries=70000):
def create_csv_file(filename="products.csv", entries=1000):
"""Erstellt CSV-Datei mit angegebener Anzahl Einträge."""
print(f"Erstelle CSV-Datei mit {entries} Einträgen...")

2
go.mod
View File

@ -7,7 +7,7 @@ require (
gittea.marcokittel.de/elio/eliotools/tools v0.0.0-20250628215830-0e90b68e2239
)
require gittea.marcokittel.de/elio/eliotools/eliofile v0.0.0-20250629091804-3a0cfb29665c
require gittea.marcokittel.de/elio/eliotools/eliofile v0.0.0-20250629160214-3393adf5f8c9
require (
filippo.io/edwards25519 v1.1.0 // indirect

2
go.sum
View File

@ -32,6 +32,8 @@ gittea.marcokittel.de/elio/eliotools/eliofile v0.0.0-20250629091158-8b052bc038c5
gittea.marcokittel.de/elio/eliotools/eliofile v0.0.0-20250629091158-8b052bc038c5/go.mod h1:h9FfBWQD/1+fF3sFaYN89K3xH54t3LHKBKQj8YiAjnU=
gittea.marcokittel.de/elio/eliotools/eliofile v0.0.0-20250629091804-3a0cfb29665c h1:W2WElxaKxa4y32IQtVHM6ziS3tnZlCDUMUiMSxHPE60=
gittea.marcokittel.de/elio/eliotools/eliofile v0.0.0-20250629091804-3a0cfb29665c/go.mod h1:h9FfBWQD/1+fF3sFaYN89K3xH54t3LHKBKQj8YiAjnU=
gittea.marcokittel.de/elio/eliotools/eliofile v0.0.0-20250629160214-3393adf5f8c9 h1:SKkk0DIJdFpdFrKBP1ZX1YizpdsCT4iI7Uhq3RGfX6U=
gittea.marcokittel.de/elio/eliotools/eliofile v0.0.0-20250629160214-3393adf5f8c9/go.mod h1:h9FfBWQD/1+fF3sFaYN89K3xH54t3LHKBKQj8YiAjnU=
gittea.marcokittel.de/elio/eliotools/logger v0.0.0-20250624203334-69cf94bf1eef h1:EzFzLINpiq712X2/t8ZMTLoWuqA6sRmpH3J0VpFi2Cg=
gittea.marcokittel.de/elio/eliotools/logger v0.0.0-20250624203334-69cf94bf1eef/go.mod h1:WDmnG6o72HhGTSkgwj2kXYcRL2MaNFNsKNBeTh6NIzo=
gittea.marcokittel.de/elio/eliotools/logger v0.0.0-20250628153134-531bb5ba4145 h1:BgHWRbDKV+lGs3T9HIa8YYy5redu6MGHTcx8es4lQdc=

67
internal/database/database.go
View File

@ -38,8 +38,7 @@ func (d *DatabaseWriter) connectDB() (*sql.DB, error) {
return db, nil
}
func (d *DatabaseWriter) createTablesIfNotExist() error {
func (d *DatabaseWriter) createWarehouseTableIfNotExist() error {
_, err := d.db.Exec(`
CREATE TABLE IF NOT EXISTS warehouseproducts (
id INT AUTO_INCREMENT PRIMARY KEY,
@ -49,11 +48,25 @@ func (d *DatabaseWriter) createTablesIfNotExist() error {
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT location_product_must_be_one UNIQUE (warehouse, productid)
) ENGINE=InnoDB;
`)
return err
}
func (d *DatabaseWriter) createDeliveryTableIfNotExist() error {
_, err := d.db.Exec(`
CREATE TABLE IF NOT EXISTS deliverytimes (
id INT AUTO_INCREMENT PRIMARY KEY,
fromcountry varchar(4) NOT NULL,
tocountry varchar(4) NOT NULL,
state varchar(4) NULL,
delivery INT DEFAULT 0,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT delivery_from_to_country_must_be_one UNIQUE (fromcountry, tocountry)
) ENGINE=InnoDB;`)
return err
}
func NewDatabaseWriter(wg *sync.WaitGroup) *DatabaseWriter {
db := DatabaseWriter{log: logger.NewMarcoLogger(),
wg: wg,
@ -64,9 +77,14 @@ func NewDatabaseWriter(wg *sync.WaitGroup) *DatabaseWriter {
os.Exit(1)
}
db.db = sql
err = db.createTablesIfNotExist()
err = db.createWarehouseTableIfNotExist()
if err != nil {
fmt.Printf("Datenbank erstellung fehlgeschlagen. %w", err)
fmt.Printf("Warenhaus-Tabellen-Erstellung fehlgeschlagen. %w", err)
os.Exit(1)
}
err = db.createDeliveryTableIfNotExist()
if err != nil {
fmt.Printf("Delivery-Tablellen-Erstellung fehlgeschlagen. %w", err)
os.Exit(1)
}
return &db
@ -85,12 +103,32 @@ func (d *DatabaseWriter) UpdateOrInsertWarehouseProduct(warehouse string, produc
return nil
}
func (db *DatabaseWriter) HandleData(ctx context.Context, data eliofile.CountryCsvData) error {
// a.log.Info("HandleData")
if strings.Contains(strings.Join(data.Data, ","), "product_id") {
func (d *DatabaseWriter) UpdateOrInsertDelivery(fromcountry string, tocountry string, state string, delivery int) error {
_, err := d.db.Exec(`
INSERT INTO deliverytimes (fromcountry, tocountry, state, delivery)
VALUES (?, ?, ?, ?)
ON DUPLICATE KEY UPDATE
fromcountry = VALUES(fromcountry),
tocountry = VALUES(tocountry),
state = VALUES(state),
delivery = VALUES(delivery)`,
fromcountry, tocountry, state, delivery)
if err != nil {
return fmt.Errorf("upsert failed: %w", err)
}
return nil
}
func (db *DatabaseWriter) HandleData(ctx context.Context, data eliofile.CountryCsvData) error {
// a.log.Info("HandleData")
if strings.Contains(strings.Join(data.Data, ","), "product_id") || strings.Contains(strings.Join(data.Data, ","), "delivery_time") {
return nil
}
//Todo: Csv reparieren.
switch data.Type {
case "stock":
{
menge := strings.Split(data.Data[0], ";")
amount, err := strconv.Atoi(menge[1])
if err != nil {
@ -98,5 +136,18 @@ func (db *DatabaseWriter) HandleData(ctx context.Context, data eliofile.CountryC
return err
}
db.UpdateOrInsertWarehouseProduct(data.CountryID, menge[0], amount)
}
case "delivery":
{
menge := strings.Split(data.Data[0], ";")
amount, err := strconv.Atoi(menge[2])
if err != nil {
fmt.Println(err)
return err
}
db.UpdateOrInsertDelivery(data.CountryID, menge[0], menge[1], amount)
}
}
return nil
}

17
internal/dataservice/dataservice.go
View File

@ -6,6 +6,7 @@ import (
"fmt"
"io/fs"
"os"
"strings"
"sync"
"time"
@ -19,6 +20,14 @@ import (
// Rückgabe eines Closures zur Verbeitung einer CSV Datei. Posix Only. No Windows.
func DataServiceDebugHandler(log logger.Logger, baseDir string, processedDir string) func(ctx context.Context, filename string, data chan<- eliofile.CountryCsvData) bool {
assignDataType := func(csvType *eliofile.CountryCsvData, filename string) {
if strings.Contains(filename, "stock") {
csvType.Type = "stock"
} else if strings.Contains(filename, "delivery") {
csvType.Type = "delivery"
}
}
return func(ctx context.Context, filename string, data chan<- eliofile.CountryCsvData) bool {
log.Infof("%s %s \n", baseDir, processedDir)
if !tools.IsFilenameValid(filename) {
@ -47,7 +56,11 @@ func DataServiceDebugHandler(log logger.Logger, baseDir string, processedDir str
if err != nil {
break
}
data <- eliofile.CountryCsvData{CountryID: filename[26:28], Data: record}
//Todo: In Tools Auslagern
entity := eliofile.CountryCsvData{CountryID: filename[26:28], Data: record}
assignDataType(&entity, filename)
data <- entity
}
err = os.Rename(baseDir+"/"+filename, processedDir+"/"+filename)
if err != nil {
@ -85,7 +98,7 @@ func (d *DataService) Run(ctx context.Context, wg *sync.WaitGroup) {
case <-ctx.Done():
return
default:
data := make(chan eliofile.CountryCsvData, 100)
data := make(chan eliofile.CountryCsvData, 1)
go func() {
defer close(data)
err := d.ef.ScanCsv(ctx, data)