Skip to content

Commit

Permalink
refactoring csv data reading and writing; update java dependencies
Browse files Browse the repository at this point in the history
  • Loading branch information
@swoellauer
swoellauer committed Jul 23, 2024
1 parent e3a9e6c commit 44123fc
Show file tree
Hide file tree
Showing 9 changed files with 255 additions and 106 deletions.
2 changes: 1 addition & 1 deletion PhotoApp/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ export NODE_OPTIONS=--openssl-legacy-provider
```

On Windows, run before "quasar dev" or "quasar build":
```bash
```Batchfile
set NODE_OPTIONS=--openssl-legacy-provider
```

Expand Down
6 changes: 3 additions & 3 deletions build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ dependencies {

implementation group: 'org.json', name: 'json', version: '20240303'
implementation group: 'org.yaml', name: 'snakeyaml', version: '1.33' // TODO: check compatibility with newer version 2.x
implementation group: 'de.siegmar', name: 'fastcsv', version: '2.2.2'
implementation group: 'de.siegmar', name: 'fastcsv', version: '3.2.0'
implementation group: 'com.opencsv', name: 'opencsv', version: '5.9'
implementation group: 'ch.randelshofer', name: 'fastdoubleparser', version: '1.0.0'

Expand All @@ -75,14 +75,14 @@ dependencies {
runtimeOnly group: 'io.jsonwebtoken', name: 'jjwt-impl', version: '0.11.5'
runtimeOnly group: 'io.jsonwebtoken', name: 'jjwt-jackson', version: '0.11.5'

implementation group: 'com.webauthn4j', name: 'webauthn4j-core', version: '0.21.7.RELEASE'
implementation group: 'com.webauthn4j', name: 'webauthn4j-core', version: '0.25.0.RELEASE'

implementation group: 'ar.com.hjg', name: 'pngj', version: '2.1.0'
implementation group: 'org.imgscalr', name: 'imgscalr-lib', version: '4.2'

implementation group: 'com.drewnoakes', name: 'metadata-extractor', version: '2.19.0'

implementation group: 'com.h2database', name: 'h2', version: '2.2.224' // old database v2.1.x files not compatible with v2.2.x
implementation group: 'com.h2database', name: 'h2', version: '2.3.230' // old database v2.1.x files not compatible with v2.2.x

implementation group: 'org.reflections', name: 'reflections', version: '0.10.2'

Expand Down
179 changes: 137 additions & 42 deletions src/audio/LabelStore.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import java.io.File;
import java.io.IOException;
import java.nio.charset.Charset;
import java.nio.file.Path;
import java.sql.ResultSet;
import java.sql.SQLException;
Expand All @@ -10,7 +11,14 @@
import org.tinylog.Logger;

import audio.LabelStoreConnector.TlLabelStoreConnector;
import de.siegmar.fastcsv.reader.CommentStrategy;
import de.siegmar.fastcsv.reader.CsvReader;
import de.siegmar.fastcsv.reader.CsvRecord;
import de.siegmar.fastcsv.reader.CsvRecordHandler;
import de.siegmar.fastcsv.writer.CsvWriter;
import de.siegmar.fastcsv.writer.LineDelimiter;
import de.siegmar.fastcsv.writer.QuoteStrategies;
import de.siegmar.fastcsv.writer.QuoteStrategy;
import util.AudioTimeUtil;

public class LabelStore {
Expand All @@ -27,11 +35,11 @@ public LabelStore(Broker broker) {
public void rebuild() {
LabelStoreConnector conn = tlLabelStoreConnector.get();
conn.init(true);

Path root_data_path = broker.config().audioConfig.root_data_path;

broker.sampleStorage().forEachOrderedSample(sample -> {
//broker.sampleManager().forEach(sample -> {
//broker.sampleManager().forEach(sample -> {
Path path = root_data_path.relativize(sample.samplePath);
int sampleMapId = conn.getOrCreateIdBySample(path.toString());
Logger.info(sample.id);
Expand Down Expand Up @@ -78,8 +86,16 @@ public void rebuild() {
}
Path output_path = output_file.toPath();

try (CsvWriter csv = CsvWriter.builder().build(output_path.resolve("generator_label.csv"))) {
csv.writeRow("location", "time", "label", "reliability", "start", "end", "sample");
try (
CsvWriter csv = CsvWriter.builder()
.fieldSeparator(',')
.quoteCharacter('"')
.quoteStrategy(null) // quote when needed only
.commentCharacter('#')
.lineDelimiter(LineDelimiter.CRLF)
.build(output_path.resolve("generator_label.csv"))
) {
csv.writeRecord("location", "time", "label", "reliability", "start", "end", "sample");
conn.forEachGeneratorLabel((int id, int label, float reliability, int location, int time, float start, float end) -> {
String labelName = conn.getLabelById(label);
String reliabilityName = "" + reliability;
Expand All @@ -88,14 +104,22 @@ public void rebuild() {
String startName = "" + start;
String endName = "" + end;
String sampleName = conn.getSampleById(id);
csv.writeRow(locationName, timeName, labelName, reliabilityName , startName, endName, sampleName);
csv.writeRecord(locationName, timeName, labelName, reliabilityName , startName, endName, sampleName);
});
} catch (IOException e) {
Logger.warn(e);
}

try (CsvWriter csv = CsvWriter.builder().build(output_path.resolve("user_label.csv"))) {
csv.writeRow("location", "time", "label", "start", "end", "creator", "creation_time", "sample");
try (
CsvWriter csv = CsvWriter.builder()
.fieldSeparator(',')
.quoteCharacter('"')
.quoteStrategy(null) // quote when needed only
.commentCharacter('#')
.lineDelimiter(LineDelimiter.CRLF)
.build(output_path.resolve("user_label.csv"))
) {
csv.writeRecord("location", "time", "label", "start", "end", "creator", "creation_time", "sample");
conn.forEachUserLabel((int id, int label, int location, int time, float start, float end, int creator, int creation_time) -> {
String labelName = conn.getLabelById(label);
String locationName = conn.getLocationById(location);
Expand All @@ -105,116 +129,187 @@ public void rebuild() {
String creatorName = conn.getCreatorById(creator);
String creationTimeName = AudioTimeUtil.ofAudiotime(creation_time).toString();
String sampleName = conn.getSampleById(id);
csv.writeRow(locationName, timeName, labelName, startName, endName, creatorName, creationTimeName, sampleName);
csv.writeRecord(locationName, timeName, labelName, startName, endName, creatorName, creationTimeName, sampleName);
});
} catch (IOException e) {
Logger.warn(e);
}

try (CsvWriter csv = CsvWriter.builder().build(output_path.resolve("location.csv"))) {
csv.writeRow("location");
try (
CsvWriter csv = CsvWriter.builder()
.fieldSeparator(',')
.quoteCharacter('"')
.quoteStrategy(null) // quote when needed only
.commentCharacter('#')
.lineDelimiter(LineDelimiter.CRLF)
.build(output_path.resolve("location.csv"))
) {
csv.writeRecord("location");
conn.forEachLocation((int id, String location) -> {
csv.writeRow(location);
csv.writeRecord(location);
});
} catch (IOException e) {
Logger.warn(e);
}

try (CsvWriter csv = CsvWriter.builder().build(output_path.resolve("label.csv"))) {
csv.writeRow("label");
try (
CsvWriter csv = CsvWriter.builder()
.fieldSeparator(',')
.quoteCharacter('"')
.quoteStrategy(null) // quote when needed only
.commentCharacter('#')
.lineDelimiter(LineDelimiter.CRLF)
.build(output_path.resolve("label.csv"))
) {
csv.writeRecord("label");
conn.forEachLabel((int id, String label) -> {
csv.writeRow(label);
csv.writeRecord(label);
});
} catch (IOException e) {
Logger.warn(e);
}

try (CsvWriter csv = CsvWriter.builder().build(output_path.resolve("creator.csv"))) {
csv.writeRow("creator");
try (
CsvWriter csv = CsvWriter.builder()
.fieldSeparator(',')
.quoteCharacter('"')
.quoteStrategy(null) // quote when needed only
.commentCharacter('#')
.lineDelimiter(LineDelimiter.CRLF)
.build(output_path.resolve("creator.csv"))
) {
csv.writeRecord("creator");
conn.forEachCreator((int id, String creator) -> {
csv.writeRow(creator);
csv.writeRecord(creator);
});
} catch (IOException e) {
Logger.warn(e);
}

try (CsvWriter csv = CsvWriter.builder().build(output_path.resolve("user_label_label.csv"))) {
csv.writeRow("label", "count");

try (
CsvWriter csv = CsvWriter.builder()
.fieldSeparator(',')
.quoteCharacter('"')
.quoteStrategy(null) // quote when needed only
.commentCharacter('#')
.lineDelimiter(LineDelimiter.CRLF)
.build(output_path.resolve("user_label_label.csv"))
) {
csv.writeRecord("label", "count");
ResultSet res = conn.conn.createStatement().executeQuery("SELECT LABEL, COUNT(*) FROM USER_LABEL_STORE GROUP BY LABEL ORDER BY COUNT(*) DESC");
while(res.next()) {
int label = res.getInt(1);
int count = res.getInt(2);
String labelName = conn.getLabelById(label);
csv.writeRow(labelName, ""+count);
csv.writeRecord(labelName, ""+count);
}
} catch (SQLException | IOException e) {
Logger.warn(e);
}

try (CsvWriter csv = CsvWriter.builder().build(output_path.resolve("user_label_location.csv"))) {
csv.writeRow("location", "count");

try (
CsvWriter csv = CsvWriter.builder()
.fieldSeparator(',')
.quoteCharacter('"')
.quoteStrategy(null) // quote when needed only
.commentCharacter('#')
.lineDelimiter(LineDelimiter.CRLF)
.build(output_path.resolve("user_label_location.csv"))
) {
csv.writeRecord("location", "count");
ResultSet res = conn.conn.createStatement().executeQuery("SELECT LOCATION, COUNT(*) FROM USER_LABEL_STORE GROUP BY LOCATION ORDER BY COUNT(*) DESC");
while(res.next()) {
int location = res.getInt(1);
int count = res.getInt(2);
String locationName = conn.getLocationById(location);
csv.writeRow(locationName, ""+count);
csv.writeRecord(locationName, ""+count);
}
} catch (SQLException | IOException e) {
Logger.warn(e);
}

try (CsvWriter csv = CsvWriter.builder().build(output_path.resolve("user_label_creator.csv"))) {
csv.writeRow("creator", "count");

try (
CsvWriter csv = CsvWriter.builder()
.fieldSeparator(',')
.quoteCharacter('"')
.quoteStrategy(null) // quote when needed only
.commentCharacter('#')
.lineDelimiter(LineDelimiter.CRLF)
.build(output_path.resolve("user_label_creator.csv"))
) {
csv.writeRecord("creator", "count");
ResultSet res = conn.conn.createStatement().executeQuery("SELECT CREATOR, COUNT(*) FROM USER_LABEL_STORE GROUP BY CREATOR ORDER BY COUNT(*) DESC");
while(res.next()) {
int creator = res.getInt(1);
int count = res.getInt(2);
String creatorName = conn.getCreatorById(creator);
csv.writeRow(creatorName, ""+count);
csv.writeRecord(creatorName, ""+count);
}
} catch (SQLException | IOException e) {
Logger.warn(e);
}

try (CsvWriter csv = CsvWriter.builder().build(output_path.resolve("user_label_location_label.csv"))) {
csv.writeRow("location", "label", "count");

try (
CsvWriter csv = CsvWriter.builder()
.fieldSeparator(',')
.quoteCharacter('"')
.quoteStrategy(null) // quote when needed only
.commentCharacter('#')
.lineDelimiter(LineDelimiter.CRLF)
.build(output_path.resolve("user_label_location_label.csv"))
) {
csv.writeRecord("location", "label", "count");
ResultSet res = conn.conn.createStatement().executeQuery("SELECT LOCATION, LABEL, COUNT(*) FROM USER_LABEL_STORE GROUP BY LOCATION, LABEL ORDER BY LOCATION ASC, COUNT(*) DESC, LABEL ASC");
while(res.next()) {
int location = res.getInt(1);
int label = res.getInt(2);
int count = res.getInt(3);
String locationName = conn.getLocationById(location);
String labelName = conn.getLabelById(label);
csv.writeRow(locationName, labelName, ""+count);
csv.writeRecord(locationName, labelName, ""+count);
}
} catch (SQLException | IOException e) {
Logger.warn(e);
}

try (CsvWriter csv = CsvWriter.builder().build(output_path.resolve("user_label_label_location.csv"))) {
csv.writeRow("label", "location", "count");

try (
CsvWriter csv = CsvWriter.builder()
.fieldSeparator(',')
.quoteCharacter('"')
.quoteStrategy(null) // quote when needed only
.commentCharacter('#')
.lineDelimiter(LineDelimiter.CRLF)
.build(output_path.resolve("user_label_label_location.csv"))
) {
csv.writeRecord("label", "location", "count");
ResultSet res = conn.conn.createStatement().executeQuery("SELECT LABEL, LOCATION, COUNT(*) FROM USER_LABEL_STORE GROUP BY LABEL, LOCATION ORDER BY LABEL ASC, COUNT(*) DESC, LOCATION ASC");
while(res.next()) {
int label = res.getInt(1);
int location = res.getInt(2);
int count = res.getInt(3);
String labelName = conn.getLabelById(label);
String locationName = conn.getLocationById(location);
csv.writeRow(labelName, locationName, ""+count);
csv.writeRecord(labelName, locationName, ""+count);
}
} catch (SQLException | IOException e) {
Logger.warn(e);
}

try (CsvWriter csv = CsvWriter.builder().build(output_path.resolve("user_label_sample.csv"))) {
csv.writeRow("sample", "count");

try (
CsvWriter csv = CsvWriter.builder()
.fieldSeparator(',')
.quoteCharacter('"')
.quoteStrategy(null) // quote when needed only
.commentCharacter('#')
.lineDelimiter(LineDelimiter.CRLF).build(output_path.resolve("user_label_sample.csv"))
) {
csv.writeRecord("sample", "count");
ResultSet res = conn.conn.createStatement().executeQuery("SELECT SAMPLE, COUNT(*) FROM USER_LABEL_STORE GROUP BY SAMPLE ORDER BY COUNT(*) DESC");
while(res.next()) {
int sample = res.getInt(1);
int count = res.getInt(2);
String sampleName = conn.getSampleById(sample);
csv.writeRow(sampleName, ""+count);
csv.writeRecord(sampleName, ""+count);
}
} catch (SQLException | IOException e) {
Logger.warn(e);
Expand Down
14 changes: 11 additions & 3 deletions src/audio/server/api/QueryHandler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import audio.processing.Metrics;
import audio.processing.SampleProcessor;
import de.siegmar.fastcsv.writer.CsvWriter;
import de.siegmar.fastcsv.writer.LineDelimiter;
import util.Web;
import util.collections.vec.Vec;

Expand Down Expand Up @@ -127,13 +128,20 @@ private void handleRootPOST(Request request, HttpServletResponse response) throw
}

response.setContentType(Web.MIME_CSV);
try (CsvWriter csv = CsvWriter.builder().build(response.getWriter())) {
try (
CsvWriter csv = CsvWriter.builder()
.fieldSeparator(',')
.quoteCharacter('"')
.quoteStrategy(null) // quote when needed only
.commentCharacter('#')
.lineDelimiter(LineDelimiter.CRLF).build(response.getWriter())
) {
String[] header = new String[HEADER_META_ROWS + metrics.size()];
header[0] = "sample";
for (int i = 0; i < metrics.size(); i++) {
header[HEADER_META_ROWS + i] = metrics.get(i).name;
}
csv.writeRow(header);
csv.writeRecord(header);
process(csv, metrics);
}
}
Expand Down Expand Up @@ -194,7 +202,7 @@ private void processSample(CsvWriter csv, Sample sample, Vec<Metric> metrics) {
Vec<String[]> rows = processSample(sample, metrics);
synchronized (csv) {
for(String[] row:rows) {
csv.writeRow(row);
csv.writeRecord(row);
}
}
}
Expand Down
Loading

0 comments on commit 44123fc

Please sign in to comment.