1 package com.acumenvelocity.ath.controller;
2
3 import java.io.File;
4 import java.nio.charset.StandardCharsets;
5 import java.util.ArrayList;
6 import java.util.Date;
7 import java.util.List;
8 import java.util.Objects;
9 import java.util.UUID;
10 import java.util.stream.Collectors;
11
12 import javax.ws.rs.core.Response.Status;
13
14 import org.apache.commons.lang3.math.NumberUtils;
15 import org.apache.solr.client.solrj.SolrClient;
16 import org.apache.solr.client.solrj.SolrQuery;
17 import org.apache.solr.client.solrj.response.QueryResponse;
18 import org.apache.solr.common.SolrDocument;
19 import org.apache.solr.common.SolrDocumentList;
20 import org.apache.solr.common.SolrInputDocument;
21
22 import com.acumenvelocity.ath.common.AthUtil;
23 import com.acumenvelocity.ath.common.Const;
24 import com.acumenvelocity.ath.common.ControllerUtil;
25 import com.acumenvelocity.ath.common.JacksonUtil;
26 import com.acumenvelocity.ath.common.Log;
27 import com.acumenvelocity.ath.common.Response;
28 import com.acumenvelocity.ath.common.SolrUtil;
29 import com.acumenvelocity.ath.common.exception.AthException;
30 import com.acumenvelocity.ath.model.CreateTranslationMemorySegmentRequest;
31 import com.acumenvelocity.ath.model.PaginationInfo;
32 import com.acumenvelocity.ath.model.TmSegment;
33 import com.acumenvelocity.ath.model.TmSegmentWrapper;
34 import com.acumenvelocity.ath.model.TmSegmentsWrapper;
35 import com.acumenvelocity.ath.model.TranslationMemoryInfo;
36 import com.acumenvelocity.ath.model.TranslationMemoryInfoWrapper;
37 import com.acumenvelocity.ath.model.TranslationMemoryInfosWrapper;
38 import com.acumenvelocity.ath.model.UpdateTranslationMemorySegmentRequest;
39 import com.acumenvelocity.ath.model.x.LayeredTextX;
40 import com.acumenvelocity.ath.solr.AthIndex;
41 import com.acumenvelocity.ath.solr.tm.SolrTmFilter;
42 import com.acumenvelocity.ath.solr.tm.SolrTmWriterStep;
43 import com.fasterxml.jackson.databind.JsonNode;
44
45 import io.swagger.oas.inflector.models.RequestContext;
46 import io.swagger.oas.inflector.models.ResponseContext;
47 import net.sf.okapi.common.LocaleId;
48 import net.sf.okapi.common.Util;
49 import net.sf.okapi.common.filters.FilterUtil;
50 import net.sf.okapi.common.filters.IFilter;
51 import net.sf.okapi.common.filters.WrapMode;
52 import net.sf.okapi.common.pipelinebuilder.XBatch;
53 import net.sf.okapi.common.pipelinebuilder.XBatchItem;
54 import net.sf.okapi.common.pipelinebuilder.XParameter;
55 import net.sf.okapi.common.pipelinebuilder.XPipeline;
56 import net.sf.okapi.common.pipelinebuilder.XPipelineStep;
57 import net.sf.okapi.common.resource.ITextUnit;
58 import net.sf.okapi.common.resource.RawDocument;
59 import net.sf.okapi.common.resource.TextUnit;
60 import net.sf.okapi.filters.table.csv.CommaSeparatedValuesFilter;
61 import net.sf.okapi.filters.table.tsv.TabSeparatedValuesFilter;
62 import net.sf.okapi.steps.common.RawDocumentToFilterEventsStep;
63 import net.sf.okapi.steps.formatconversion.FormatConversionStep;
64 import net.sf.okapi.steps.formatconversion.Parameters;
65
66 public class TranslationMemoryController {
67 private static final String TMX_FILE_NAME = "download.tmx";
68
69 public ResponseContext getTranslationMemories(RequestContext request, Integer page,
70 Integer pageSize) {
71 try {
72 SolrClient solrClient = AthIndex.getSolr().getClient();
73
74
75 SolrQuery q = new SolrQuery("*:*")
76 .setFields(Const.ATH_PROP_TM_ID)
77 .addSort("_docid_", SolrQuery.ORDER.asc)
78 .setRows(Integer.MAX_VALUE);
79
80 QueryResponse response = solrClient.query(Const.SOLR_CORE_ATH_TMS, q);
81
82 List<String> allTmIds = response.getResults().stream()
83 .map(doc -> doc.getFieldValue(Const.ATH_PROP_TM_ID))
84 .filter(Objects::nonNull)
85 .map(Object::toString)
86 .filter(id -> !id.isEmpty())
87 .distinct()
88 .collect(Collectors.toList());
89
90 TranslationMemoryInfosWrapper wrapper = new TranslationMemoryInfosWrapper();
91
92
93 if (allTmIds.isEmpty()) {
94 wrapper.translationMemories(new ArrayList<>())
95 .pagination(new PaginationInfo()
96 .page(1)
97 .pageSize(0)
98 .totalItems(0L)
99 .totalPages(0)
100 .hasNext(false)
101 .hasPrevious(false));
102
103 return Response.success(200, wrapper);
104 }
105
106 long totalItems = allTmIds.size();
107 int size = (pageSize != null) ? Math.max(1, Math.min(100, pageSize)) : (int) totalItems;
108 int totalPages = (int) Math.ceil(totalItems / (double) size);
109
110 int pageNum;
111 List<String> tmIdsToProcess;
112
113 if (page == null && pageSize == null) {
114
115 pageNum = 1;
116 int end = Math.min(size, allTmIds.size());
117 tmIdsToProcess = allTmIds.subList(0, end);
118
119 } else {
120 pageNum = (page != null) ? Math.max(1, Math.min(page, Math.max(1, totalPages))) : 1;
121 int start = (pageNum - 1) * size;
122 int end = Math.min(start + size, allTmIds.size());
123 tmIdsToProcess = allTmIds.subList(start, end);
124 }
125
126 List<TranslationMemoryInfo> resultTmInfos = tmIdsToProcess.stream()
127 .map(this::getTranslationMemoryInfoInternal)
128 .filter(Objects::nonNull)
129 .collect(Collectors.toList());
130
131 PaginationInfo pagination = new PaginationInfo()
132 .page(pageNum)
133 .pageSize(size)
134 .totalItems(totalItems)
135 .totalPages(totalPages)
136 .hasNext(pageNum < totalPages)
137 .hasPrevious(pageNum > 1);
138
139 wrapper.translationMemories(resultTmInfos)
140 .pagination(pagination);
141
142 return Response.success(200, wrapper);
143
144 } catch (Exception e) {
145 return Response.error(500, e, "Error fetching translation memories");
146 }
147 }
148
149 private TranslationMemoryInfo getTranslationMemoryInfoInternal(String tmId) {
150 try {
151 SolrClient solrClient = AthIndex.getSolr().getClient();
152 String query = Log.format("tmId:\"{}\"", tmId);
153 SolrQuery solrQuery = new SolrQuery(query);
154 solrQuery.setRows(1);
155 solrQuery.setFields(Const.ATH_PROP_TM_ID, Const.ATH_PROP_TM_FILE_NAME,
156 Const.ATH_PROP_SRC_LANG,
157 Const.ATH_PROP_TRG_LANG, Const.ATH_PROP_CREATED_BY, Const.ATH_PROP_CREATED_AT,
158 Const.ATH_PROP_UPDATED_BY, Const.ATH_PROP_UPDATED_AT);
159
160 QueryResponse response = solrClient.query(Const.SOLR_CORE_ATH_TMS, solrQuery);
161 SolrDocumentList docList = response.getResults();
162
163 if (docList.isEmpty()) {
164 return null;
165 }
166
167 SolrDocument doc = docList.get(0);
168 TranslationMemoryInfo tmInfo = new TranslationMemoryInfo();
169
170 tmInfo.setTmId(
171 AthUtil.safeToUuid(SolrUtil.safeGetField(doc, Const.ATH_PROP_TM_ID, null), null));
172
173 tmInfo.setTmFileName(SolrUtil.safeGetField(doc, Const.ATH_PROP_TM_FILE_NAME, null));
174 tmInfo.setSrcLang(SolrUtil.safeGetField(doc, Const.ATH_PROP_SRC_LANG, null));
175 tmInfo.setTrgLang(SolrUtil.safeGetField(doc, Const.ATH_PROP_TRG_LANG, null));
176
177 tmInfo.setCreatedBy(
178 AthUtil.safeToUuid(SolrUtil.safeGetField(doc, Const.ATH_PROP_CREATED_BY, null), null));
179
180 tmInfo.setCreatedAt(AthUtil.safeToDate(doc.get(Const.ATH_PROP_CREATED_AT), null));
181
182 tmInfo.setUpdatedBy(
183 AthUtil.safeToUuid(SolrUtil.safeGetField(doc, Const.ATH_PROP_UPDATED_BY, null), null));
184
185 tmInfo.setUpdatedAt(AthUtil.safeToDate(doc.get(Const.ATH_PROP_UPDATED_AT), null));
186
187
188 long segmentsCount = SolrUtil.getNumDocuments(Const.SOLR_CORE_ATH_TM_SEGMENTS, query);
189 tmInfo.setSegmentsCount(segmentsCount);
190
191 return tmInfo;
192
193 } catch (Exception e) {
194 return null;
195 }
196 }
197
198 public ResponseContext createTranslationMemory(RequestContext request, UUID tmId,
199 File tmFile, String tmFileName, Integer tmFileStartLine, String tmFileDelimiter,
200 Integer tmFileSrcColumn, Integer tmFileTrgColumn, String tmSrcLang, String tmTrgLang,
201 UUID userId) {
202
203 if (!ControllerUtil.checkParam(tmId)) {
204 return Response.error(400, "Invalid request parameter TM Id: " + tmId);
205 }
206
207 if (!ControllerUtil.checkParam(tmFile)) {
208 return Response.error(400, "Invalid request, tmFile object is null");
209 }
210
211 if (!ControllerUtil.checkParam(tmFileName)) {
212 return Response.error(400, "Invalid request, tmFileName is not specified");
213 }
214
215 if (!ControllerUtil.checkParam(tmSrcLang)) {
216 return Response.error(400, "Invalid request, tmSrcLang is not specified");
217 }
218
219 if (!ControllerUtil.checkParam(tmTrgLang)) {
220 return Response.error(400, "Invalid request, tmTrgLang is not specified");
221 }
222
223 if (!ControllerUtil.checkParam(userId)) {
224 return Response.error(400, "Invalid request parameter User Id: " + userId);
225 }
226
227
228 String query = Log.format("tmId:\"{}\"", tmId);
229 try {
230 if (SolrUtil.getNumDocuments(Const.SOLR_CORE_ATH_TMS, query) > 0) {
231 return Response.error(400, "Translation Memory already exists, tmId: " + tmId);
232 }
233 } catch (Exception e) {
234 return Response.error(500, "Error checking TM existence -- " + e.getMessage());
235 }
236
237 LocaleId srcLoc = LocaleId.fromString(tmSrcLang);
238 LocaleId trgLoc = LocaleId.fromString(tmTrgLang);
239
240 IFilter filter = getFilter(tmFileName, tmFileStartLine, tmFileDelimiter, tmFileSrcColumn,
241 tmFileTrgColumn);
242
243 try {
244
245 SolrInputDocument tmDoc = new SolrInputDocument();
246
247 tmDoc.addField(Const.ATH_PROP_TM_ID, tmId.toString());
248 tmDoc.addField(Const.ATH_PROP_TM_FILE_NAME, tmFileName);
249 tmDoc.addField(Const.ATH_PROP_SRC_LANG, tmSrcLang);
250 tmDoc.addField(Const.ATH_PROP_TRG_LANG, tmTrgLang);
251 tmDoc.addField(Const.ATH_PROP_CREATED_BY, userId.toString());
252 tmDoc.addField(Const.ATH_PROP_CREATED_AT, new Date());
253 tmDoc.addField(Const.ATH_PROP_UPDATED_BY, userId.toString());
254 tmDoc.addField(Const.ATH_PROP_UPDATED_AT, new Date());
255
256 SolrClient solrClient = AthIndex.getSolr().getClient();
257 solrClient.add(Const.SOLR_CORE_ATH_TMS, tmDoc);
258 solrClient.commit(Const.SOLR_CORE_ATH_TMS);
259
260
261 try (XPipeline pl = new XPipeline("TM export",
262 new XBatch(
263 new XBatchItem(
264 Util.toURI(tmFile.getAbsolutePath()),
265 StandardCharsets.UTF_8.name(),
266 srcLoc, trgLoc)),
267 new RawDocumentToFilterEventsStep(filter),
268 new SolrTmWriterStep(tmId, tmFileName, userId, true))) {
269
270 pl.execute();
271
272 } catch (Exception e) {
273
274 solrClient.deleteByQuery(Const.SOLR_CORE_ATH_TMS, query);
275 solrClient.commit(Const.SOLR_CORE_ATH_TMS);
276 throw e;
277 }
278
279 } catch (Exception e) {
280 return Response.error(500, "Error creating TM -- " + e.getMessage());
281 }
282
283 return Response.success(201, "Success");
284 }
285
286 private IFilter getFilter(String tmFileName, Integer tmFileStartLine, String tmFileDelimiter,
287 Integer tmFileSrcColumn, Integer tmFileTrgColumn) {
288
289 String fileExt = com.google.common.io.Files.getFileExtension(tmFileName);
290
291 if ("tmx".equalsIgnoreCase(fileExt)) {
292 return FilterUtil.createFilter("okf_tmx");
293
294 } else {
295 net.sf.okapi.filters.table.csv.Parameters csvParams = new net.sf.okapi.filters.table.csv.Parameters();
296
297 csvParams.textQualifier = "\"";
298 csvParams.removeQualifiers = true;
299 csvParams.escapingMode = net.sf.okapi.filters.table.csv.Parameters.ESCAPING_MODE_DUPLICATION;
300 csvParams.addQualifiers = false;
301 csvParams.columnNamesLineNum = 0;
302 csvParams.valuesStartLineNum = tmFileStartLine;
303 csvParams.detectColumnsMode = net.sf.okapi.filters.table.csv.Parameters.DETECT_COLUMNS_NONE;
304 csvParams.numColumns = NumberUtils.max(tmFileSrcColumn, tmFileTrgColumn);
305 csvParams.sendHeaderMode = net.sf.okapi.filters.table.csv.Parameters.SEND_HEADER_NONE;
306 csvParams.trimMode = net.sf.okapi.filters.table.csv.Parameters.TRIM_NONQUALIFIED_ONLY;
307 csvParams.sendColumnsMode = net.sf.okapi.filters.table.csv.Parameters.SEND_COLUMNS_LISTED;
308 csvParams.sourceIdColumns = "";
309 csvParams.sourceColumns = tmFileSrcColumn == 0 ? "" : String.valueOf(tmFileSrcColumn);
310 csvParams.targetColumns = tmFileTrgColumn == 0 ? "" : String.valueOf(tmFileTrgColumn);
311 csvParams.commentColumns = "";
312 csvParams.commentSourceRefs = csvParams.sourceColumns;
313 csvParams.recordIdColumn = 0;
314 csvParams.sourceIdSourceRefs = "";
315 csvParams.sourceIdSuffixes = "";
316 csvParams.targetLanguages = "";
317 csvParams.targetSourceRefs = csvParams.sourceColumns;
318 csvParams.trimLeading = true;
319 csvParams.trimTrailing = true;
320 csvParams.preserveWS = true;
321 csvParams.useCodeFinder = false;
322 csvParams.wrapMode = WrapMode.NONE;
323 csvParams.subfilter = null;
324
325 if ("tsv".equalsIgnoreCase(fileExt)) {
326 csvParams.fieldDelimiter = Util.isEmpty(tmFileDelimiter) ? "\t" : tmFileDelimiter;
327
328 TabSeparatedValuesFilter tsvFilter = new TabSeparatedValuesFilter();
329 tsvFilter.setParameters(csvParams);
330 return tsvFilter;
331
332 } else {
333 csvParams.fieldDelimiter = Util.isEmpty(tmFileDelimiter) ? "," : tmFileDelimiter;
334
335 CommaSeparatedValuesFilter csvFilter = new CommaSeparatedValuesFilter();
336 csvFilter.setParameters(csvParams);
337 return csvFilter;
338 }
339 }
340 }
341
342 public ResponseContext exportTranslationMemory(RequestContext request, UUID tmId) {
343 if (!ControllerUtil.checkParam(tmId)) {
344 return Response.error(400, "Invalid request parameter TM Id: " + tmId);
345 }
346
347 String query = Log.format("tmId:\"{}\"", tmId);
348
349 try {
350 if (SolrUtil.getNumDocuments(Const.SOLR_CORE_ATH_TMS, query) <= 0) {
351 return Response.error(404, "TM not found, tmId: " + tmId);
352 }
353
354 File file = null;
355
356 try {
357 file = AthUtil.createTempFile();
358
359 try (SolrTmFilter tmFilter = new SolrTmFilter(AthIndex.getSolr().getClient(),
360 Const.SOLR_CORE_ATH_TM_SEGMENTS, tmId)) {
361
362 LocaleId srcLoc = null;
363 LocaleId trgLoc = null;
364
365 TranslationMemoryInfo tmInfo = getTranslationMemoryInfoInternal(tmId.toString());
366
367 try {
368 srcLoc = LocaleId.fromString(tmInfo.getSrcLang());
369 trgLoc = LocaleId.fromString(tmInfo.getTrgLang());
370
371 } catch (Exception e) {
372 Log.warn(getClass(), "Error detecting TM locales: tmId='{}' -- {}", tmId,
373 e.getMessage());
374 }
375
376 try (XPipeline pl = new XPipeline("TM export",
377 new XBatch(
378 new XBatchItem(
379 new RawDocument(
380 "{}",
381 srcLoc == null ? LocaleId.AUTODETECT : srcLoc,
382 trgLoc == null ? LocaleId.AUTODETECT : trgLoc))),
383
384 new RawDocumentToFilterEventsStep(tmFilter),
385
386 new XPipelineStep(FormatConversionStep.class,
387 new XParameter("outputFormat", Parameters.FORMAT_TMX),
388 new XParameter("outputPath", file.toURI().getPath()),
389 new XParameter("singleOutput", true),
390 new XParameter("overwriteSameSource", false)))) {
391
392 pl.execute();
393 }
394 }
395
396 } catch (Exception e) {
397 return Response.error(500, "Cannot create temp file -- " + e.getMessage());
398 }
399
400 return Response.builder()
401 .status(Status.OK)
402 .header("Content-Disposition", Log.format("attachment; filename=\"{}\"", TMX_FILE_NAME))
403 .entity(file)
404 .build();
405
406 } catch (Exception e) {
407 return Response.error(500, "Error exporting TM -- " + e.getMessage());
408 }
409 }
410
411 public ResponseContext updateTranslationMemory(RequestContext request, UUID tmId,
412 File tmFile, String tmFileName, Integer tmFileStartLine, String tmFileDelimiter,
413 Integer tmFileSrcColumn, Integer tmFileTrgColumn, UUID userId) {
414
415 if (!ControllerUtil.checkParam(tmId)) {
416 return Response.error(400, "Invalid request parameter TM Id: " + tmId);
417 }
418
419 if (!ControllerUtil.checkParam(tmFile)) {
420 return Response.error(400, "Invalid request, tmFile object is null");
421 }
422
423 if (!ControllerUtil.checkParam(tmFileName)) {
424 return Response.error(400, "Invalid request, tmFileName is not specified");
425 }
426
427 if (!ControllerUtil.checkParam(userId)) {
428 return Response.error(400, "Invalid request parameter User Id: " + userId);
429 }
430
431 String query = Log.format("tmId:\"{}\"", tmId);
432
433 try {
434 if (SolrUtil.getNumDocuments(Const.SOLR_CORE_ATH_TMS, query) <= 0) {
435 return Response.error(404, "TM not found, tmId: " + tmId);
436 }
437
438 LocaleId srcLoc = LocaleId.ENGLISH;
439 LocaleId trgLoc = LocaleId.FRENCH;
440
441
442 TranslationMemoryInfo tmInfo = getTranslationMemoryInfoInternal(tmId.toString());
443
444 if (tmInfo != null) {
445 srcLoc = LocaleId.fromString(tmInfo.getSrcLang());
446 trgLoc = LocaleId.fromString(tmInfo.getTrgLang());
447 }
448
449 IFilter filter = getFilter(tmFileName, tmFileStartLine, tmFileDelimiter, tmFileSrcColumn,
450 tmFileTrgColumn);
451
452 try (XPipeline pl = new XPipeline("TM update",
453 new XBatch(
454 new XBatchItem(
455 Util.toURI(tmFile.getAbsolutePath()),
456 StandardCharsets.UTF_8.name(),
457 srcLoc, trgLoc)),
458 new RawDocumentToFilterEventsStep(filter),
459 new SolrTmWriterStep(tmId, tmFileName, userId, false))) {
460
461 pl.execute();
462
463
464 SolrClient solrClient = AthIndex.getSolr().getClient();
465 SolrInputDocument updateDoc = SolrUtil.toInputDocument(SolrUtil.getTmByTmId(tmId));
466
467 updateDoc.setField(Const.ATH_PROP_TM_FILE_NAME, tmFileName);
468 updateDoc.setField(Const.ATH_PROP_UPDATED_BY, userId.toString());
469 updateDoc.setField(Const.ATH_PROP_UPDATED_AT, new Date());
470
471 solrClient.add(Const.SOLR_CORE_ATH_TMS, updateDoc);
472 solrClient.commit(Const.SOLR_CORE_ATH_TMS);
473
474 } catch (Exception e) {
475 return Response.error(500, "Error updating TM -- " + e.getMessage());
476 }
477
478 return Response.success(200, "Success");
479
480 } catch (Exception e) {
481 return Response.error(500, "Error updating TM -- " + e.getMessage());
482 }
483 }
484
485 public ResponseContext deleteTranslationMemory(RequestContext request, UUID tmId) {
486 if (!ControllerUtil.checkParam(tmId)) {
487 return Response.error(400, "Invalid request parameter TM Id: " + tmId);
488 }
489
490 String query = Log.format("tmId:\"{}\"", tmId);
491
492 try {
493 if (SolrUtil.getNumDocuments(Const.SOLR_CORE_ATH_TMS, query) <= 0) {
494 return Response.error(404, "TM not found, tmId: " + tmId);
495 }
496
497
498 AthIndex.deleteByQuery(Const.SOLR_CORE_ATH_TM_SEGMENTS, query);
499 AthIndex.deleteByQuery(Const.SOLR_CORE_ATH_TMS, query);
500
501 return Response.success(204, "Translation Memory (id={}) was deleted successfully", tmId);
502
503 } catch (Exception e) {
504 String st = Log.format("Error deleting Translation Memory (id={}) -- {}",
505 tmId, e.getMessage());
506
507 Log.error(this.getClass(), e, st);
508 return Response.error(500, st);
509 }
510 }
511
512 public ResponseContext getTranslationMemoryInfo(RequestContext request, UUID tmId) {
513 if (!ControllerUtil.checkParam(tmId)) {
514 return Response.error(400, "Invalid request parameter TM Id: " + tmId);
515 }
516
517 TranslationMemoryInfo tmInfo = getTranslationMemoryInfoInternal(tmId.toString());
518
519 if (tmInfo == null) {
520 return Response.error(404, "TM not found, tmId: " + tmId);
521 }
522
523 TranslationMemoryInfoWrapper tiw = new TranslationMemoryInfoWrapper();
524 tiw.setTranslationMemoryInfo(tmInfo);
525
526 return Response.success(200, tiw);
527 }
528
529 public ResponseContext getTranslationMemorySegments(RequestContext request, UUID tmId,
530 Integer page, Integer pageSize) {
531
532 if (!ControllerUtil.checkParam(tmId)) {
533 return Response.error(400, "Invalid request parameter TM Id: " + tmId);
534 }
535
536 String query = Log.format("tmId:\"{}\"", tmId);
537
538 try {
539 long totalItems = SolrUtil.getNumDocuments(Const.SOLR_CORE_ATH_TM_SEGMENTS, query);
540
541 if (totalItems <= 0) {
542 TmSegmentsWrapper emptyWrapper = new TmSegmentsWrapper()
543 .tmSegments(new ArrayList<>())
544 .pagination(new PaginationInfo()
545 .page(1)
546 .pageSize(0)
547 .totalItems(0L)
548 .totalPages(0)
549 .hasNext(false)
550 .hasPrevious(false));
551
552 return Response.success(200, emptyWrapper);
553 }
554
555 int size = (pageSize != null) ? Math.max(1, Math.min(100, pageSize)) : (int) totalItems;
556 int totalPages = (int) Math.ceil(totalItems / (double) size);
557 int pageNum = (page != null) ? Math.max(1, Math.min(page, Math.max(1, totalPages))) : 1;
558
559 SolrClient solrClient = AthIndex.getSolr().getClient();
560 SolrQuery solrQuery = new SolrQuery(query)
561 .setStart((pageNum - 1) * size)
562 .setRows(size)
563 .addSort(Const.ATH_PROP_CREATED_AT, SolrQuery.ORDER.asc);
564
565 QueryResponse response = solrClient.query(Const.SOLR_CORE_ATH_TM_SEGMENTS, solrQuery);
566 SolrDocumentList docList = response.getResults();
567
568 List<TmSegment> segments = docList.stream()
569 .map(this::toTmSegment)
570 .filter(Objects::nonNull)
571 .collect(Collectors.toList());
572
573
574 PaginationInfo pagination = new PaginationInfo()
575 .page(pageNum)
576 .pageSize(size)
577 .totalItems(totalItems)
578 .totalPages(totalPages)
579 .hasNext(pageNum < totalPages)
580 .hasPrevious(pageNum > 1);
581
582 TmSegmentsWrapper wrapper = new TmSegmentsWrapper()
583 .tmSegments(segments)
584 .pagination(pagination);
585
586 return Response.success(200, wrapper);
587
588 } catch (Exception e) {
589 return Response.error(500, e, "Error fetching TM segments");
590 }
591 }
592
593
594
595
596
597
598
599
600
601
602 public ResponseContext createTranslationMemorySegment(RequestContext request, UUID tmId,
603 JsonNode bodyNode) {
604
605 if (!ControllerUtil.checkParam(tmId)) {
606 return Response.error(400, "Invalid request parameter TM Id: " + tmId);
607 }
608
609 if (!ControllerUtil.checkParam(bodyNode)) {
610 return Response.error(400, "Invalid request body");
611 }
612
613 CreateTranslationMemorySegmentRequest body = AthUtil.safeFromJsonNode(bodyNode,
614 CreateTranslationMemorySegmentRequest.class, null);
615
616 if (body == null) {
617 return Response.error(400, "Invalid request body");
618 }
619
620 try {
621 UUID tmSegId = body.getTmSegId();
622 LayeredTextX source = body.getSource();
623 LayeredTextX target = body.getTarget();
624 UUID docId = body.getDocId();
625 String docFileName = body.getDocFileName();
626 UUID userId = body.getUserId();
627
628 if (!ControllerUtil.checkParam(tmSegId)) {
629 return Response.error(400, "Invalid request parameter TM Segment Id: " + tmSegId);
630 }
631
632 if (!ControllerUtil.checkParam(source)) {
633 return Response.error(400, "Invalid request, source is not specified");
634 }
635
636 if (!ControllerUtil.checkParam(target)) {
637 return Response.error(400, "Invalid request, target is not specified");
638 }
639
640 if (!ControllerUtil.checkParam(userId)) {
641 return Response.error(400, "Invalid request parameter User Id: " + userId);
642 }
643
644 SolrDocument tmDoc = SolrUtil.getTmByTmId(tmId);
645
646 if (tmDoc == null) {
647 return Response.error(404, "TM not found, tmId: " + tmId);
648 }
649
650
651 String segmentSolrId = SolrUtil.buildTmSegSolrId(tmId, source.getTextWithCodes());
652
653
654 try {
655 SolrClient solrClient = AthIndex.getSolr().getClient();
656 SolrQuery checkQuery = new SolrQuery("id:\"" + segmentSolrId + "\"");
657 checkQuery.setRows(1);
658
659 QueryResponse checkResponse = solrClient.query(Const.SOLR_CORE_ATH_TM_SEGMENTS, checkQuery);
660 SolrDocumentList docs = checkResponse.getResults();
661
662 if (docs != null && !docs.isEmpty()) {
663
664 SolrDocument existingDoc = docs.get(0);
665
666 UUID existingTmSegId = AthUtil.safeToUuid(
667 SolrUtil.safeGetField(existingDoc, Const.ATH_PROP_TM_SEG_ID, null), null);
668
669 if (existingTmSegId != null) {
670 Log.info(getClass(),
671 "TM segment with source='{}' already exists (tmSegId={}), updating instead",
672 source.getTextWithCodes(), existingTmSegId);
673
674
675 UpdateTranslationMemorySegmentRequest updateBody = new UpdateTranslationMemorySegmentRequest();
676 updateBody.setTmSegId(existingTmSegId);
677 updateBody.setDocId(docId);
678 updateBody.setDocFileName(docFileName);
679 updateBody.setTarget(target);
680 updateBody.setUserId(userId);
681
682 JsonNode updateBodyNode = JacksonUtil.makeNode(updateBody);
683
684
685 return updateTranslationMemorySegment(request, tmId, existingTmSegId, updateBodyNode);
686 }
687 }
688 } catch (Exception e) {
689 Log.warn(getClass(), "Error checking for existing TM segment: {}", e.getMessage());
690
691 }
692
693
694 SolrClient solrClient = AthIndex.getSolr().getClient();
695
696 String srcLang = SolrUtil.safeGetField(tmDoc, Const.ATH_PROP_SRC_LANG, null);
697 String trgLang = SolrUtil.safeGetField(tmDoc, Const.ATH_PROP_TRG_LANG, null);
698 String tmFileName = SolrUtil.safeGetField(tmDoc, Const.ATH_PROP_TM_FILE_NAME, null);
699
700 TmSegment segment = new TmSegment();
701
702 segment.setId(segmentSolrId);
703 segment.setTmSegId(tmSegId);
704 segment.setTmId(tmId);
705 segment.setTmFileName(tmFileName);
706
707 if (docId != null) {
708 segment.setDocId(docId);
709 }
710
711 if (docFileName != null) {
712 segment.setDocFileName(docFileName);
713 }
714
715 segment.setSrcLang(srcLang);
716 segment.setTrgLang(trgLang);
717 segment.setSource(source);
718 segment.setTarget(target);
719 segment.setCreatedBy(userId);
720 segment.setCreatedAt(new Date());
721
722 SolrInputDocument solrDoc = toSolrDoc(segment);
723 solrClient.add(Const.SOLR_CORE_ATH_TM_SEGMENTS, solrDoc);
724 solrClient.commit(Const.SOLR_CORE_ATH_TM_SEGMENTS);
725
726
727 SolrInputDocument updateDoc = SolrUtil.toInputDocument(SolrUtil.getTmByTmId(tmId));
728 updateDoc.setField(Const.ATH_PROP_UPDATED_BY, userId.toString());
729 updateDoc.setField(Const.ATH_PROP_UPDATED_AT, new Date());
730
731 solrClient.add(Const.SOLR_CORE_ATH_TMS, updateDoc);
732 solrClient.commit(Const.SOLR_CORE_ATH_TMS);
733
734 return Response.success(201, "TM segment created successfully");
735
736 } catch (Exception e) {
737 return Response.error(500, "Error creating TM segment -- " + e.getMessage());
738 }
739 }
740
741 public ResponseContext getTranslationMemorySegment(RequestContext request, UUID tmId,
742 UUID tmSegId) {
743
744 if (!ControllerUtil.checkParam(tmId)) {
745 return Response.error(400, "Invalid request parameter TM Id: " + tmId);
746 }
747
748 if (!ControllerUtil.checkParam(tmSegId)) {
749 return Response.error(400, "Invalid request parameter Segment Id: " + tmSegId);
750 }
751
752 String query = Log.format("tmId:\"{}\" AND tmSegId:\"{}\"", tmId, tmSegId);
753
754 try {
755 QueryResponse response = AthIndex.getMany(Const.SOLR_CORE_ATH_TM_SEGMENTS, query, null,
756 QueryResponse.class);
757
758 SolrDocumentList docList = response.getResults();
759
760 if (docList.isEmpty()) {
761 return Response.error(404, "TM segment not found");
762 }
763
764 SolrDocument doc = docList.get(0);
765 TmSegment segment = toTmSegment(doc);
766
767 if (segment == null) {
768 return Response.error(500, "Error parsing segment data");
769 }
770
771 TmSegmentWrapper wrapper = new TmSegmentWrapper();
772 wrapper.setTmSegment(segment);
773
774 return Response.success(200, wrapper);
775
776 } catch (Exception e) {
777 return Response.error(500, "Error fetching TM segment -- " + e.getMessage());
778 }
779 }
780
781 public ResponseContext updateTranslationMemorySegment(RequestContext request, UUID tmId,
782 UUID tmSegId, JsonNode bodyNode) {
783
784 if (!ControllerUtil.checkParam(tmId)) {
785 return Response.error(400, "Invalid request parameter TM Id: " + tmId);
786 }
787
788 if (!ControllerUtil.checkParam(tmSegId)) {
789 return Response.error(400, "Invalid request parameter Segment Id: " + tmSegId);
790 }
791
792 if (!ControllerUtil.checkParam(bodyNode)) {
793 return Response.error(400, "Invalid request body");
794 }
795
796 UpdateTranslationMemorySegmentRequest body = AthUtil.safeFromJsonNode(bodyNode,
797 UpdateTranslationMemorySegmentRequest.class, null);
798
799 if (body == null) {
800 return Response.error(400, "Invalid request body");
801 }
802
803 try {
804 UUID tmSegIdFromBody = body.getTmSegId();
805 UUID docId = body.getDocId();
806 String docFileName = body.getDocFileName();
807 LayeredTextX target = body.getTarget();
808 UUID userId = body.getUserId();
809
810 if (!ControllerUtil.checkParam(target)) {
811 return Response.error(400, "Invalid request, target is not specified");
812 }
813
814 if (!ControllerUtil.checkParam(userId)) {
815 return Response.error(400, "Invalid request parameter User Id: " + userId);
816 }
817
818 if (tmSegIdFromBody != null && !tmSegIdFromBody.equals(tmSegId)) {
819 return Response.error(400, "TM Segment Id mismatch");
820 }
821
822 if (docId != null) {
823 String docQuery = Log.format("docId:\"{}\"", docId);
824 if (SolrUtil.getNumDocuments(Const.SOLR_CORE_ATH_TMS, docQuery) <= 0) {
825 return Response.error(404, "Document not found, docId: " + docId);
826 }
827 }
828
829 String query = Log.format("tmId:\"{}\" AND tmSegId:\"{}\"", tmId, tmSegId);
830
831 QueryResponse response = AthIndex.getMany(Const.SOLR_CORE_ATH_TM_SEGMENTS, query, null,
832 QueryResponse.class);
833
834 SolrDocumentList docList = response.getResults();
835
836 if (docList.isEmpty()) {
837 return Response.error(404, "TM segment not found");
838 }
839
840 SolrDocument originalDoc = docList.get(0);
841 TmSegment segment = toTmSegment(originalDoc);
842
843 segment.setTarget(target);
844 if (docId != null) {
845 segment.setDocId(docId);
846 }
847 if (docFileName != null) {
848 segment.setDocFileName(docFileName);
849 }
850 segment.setUpdatedBy(userId);
851 segment.setUpdatedAt(new Date());
852
853 SolrInputDocument solrDoc = toSolrDoc(segment);
854 SolrClient solrClient = AthIndex.getSolr().getClient();
855 solrClient.add(Const.SOLR_CORE_ATH_TM_SEGMENTS, solrDoc);
856 solrClient.commit(Const.SOLR_CORE_ATH_TM_SEGMENTS);
857
858
859 SolrInputDocument updateDoc = SolrUtil.toInputDocument(SolrUtil.getTmByTmId(tmId));
860
861 updateDoc.setField(Const.ATH_PROP_UPDATED_BY, userId.toString());
862 updateDoc.setField(Const.ATH_PROP_UPDATED_AT, new Date());
863
864 solrClient.add(Const.SOLR_CORE_ATH_TMS, updateDoc);
865 solrClient.commit(Const.SOLR_CORE_ATH_TMS);
866
867 return Response.success(200, "TM segment updated successfully");
868
869 } catch (Exception e) {
870 return Response.error(500, "Error updating TM segment -- " + e.getMessage());
871 }
872 }
873
874 public ResponseContext deleteTranslationMemorySegment(RequestContext request, UUID tmId,
875 UUID tmSegId) {
876
877 if (!ControllerUtil.checkParam(tmId)) {
878 return Response.error(400, "Invalid request parameter TM Id: " + tmId);
879 }
880
881 if (!ControllerUtil.checkParam(tmSegId)) {
882 return Response.error(400, "Invalid request parameter Segment Id: " + tmSegId);
883 }
884
885 String query = Log.format("tmId:\"{}\" AND tmSegId:\"{}\"", tmId, tmSegId);
886
887 try {
888 if (SolrUtil.getNumDocuments(Const.SOLR_CORE_ATH_TM_SEGMENTS, query) <= 0) {
889 return Response.error(404, "TM segment not found");
890 }
891
892 SolrClient solrClient = AthIndex.getSolr().getClient();
893 AthIndex.deleteByQuery(Const.SOLR_CORE_ATH_TM_SEGMENTS, query);
894 solrClient.commit(Const.SOLR_CORE_ATH_TM_SEGMENTS);
895
896
897 SolrInputDocument updateDoc = SolrUtil.toInputDocument(SolrUtil.getTmByTmId(tmId));
898
899 updateDoc.setField(Const.ATH_PROP_UPDATED_AT, new Date());
900
901 solrClient.add(Const.SOLR_CORE_ATH_TMS, updateDoc);
902 solrClient.commit(Const.SOLR_CORE_ATH_TMS);
903
904 return Response.success(204, "TM segment deleted successfully");
905
906 } catch (Exception e) {
907 String st = Log.format("Error deleting TM segment (tmId={}, segId={}) -- {}",
908 tmId, tmSegId, e.getMessage());
909
910 Log.error(this.getClass(), e, st);
911 return Response.error(500, st);
912 }
913 }
914
915
916
917 private TmSegment toTmSegment(SolrDocument doc) {
918 try {
919 TmSegment segment = new TmSegment();
920
921
922 segment.setId(SolrUtil.safeGetField(doc, Const.ATH_PROP_SOLR_ID, null));
923
924 segment.setTmSegId(AthUtil.safeToUuid(
925 SolrUtil.safeGetField(doc, Const.ATH_PROP_TM_SEG_ID, null), null));
926
927 segment.setTmId(AthUtil.safeToUuid(
928 SolrUtil.safeGetField(doc, Const.ATH_PROP_TM_ID, null), null));
929
930 segment.setTmFileName(SolrUtil.safeGetField(doc, Const.ATH_PROP_TM_FILE_NAME, null));
931
932 segment.setDocId(AthUtil.safeToUuid(
933 SolrUtil.safeGetField(doc, Const.ATH_PROP_DOC_ID, null), null));
934
935 segment.setDocFileName(SolrUtil.safeGetField(doc, Const.ATH_PROP_DOC_FILE_NAME, null));
936
937 segment.setSrcLang(SolrUtil.safeGetField(doc, Const.ATH_PROP_SRC_LANG, null));
938
939 segment.setTrgLang(SolrUtil.safeGetField(doc, Const.ATH_PROP_TRG_LANG, null));
940
941
942 String sourceJson = doc.getFieldValue(Const.ATH_PROP_SOURCE_JSON) != null
943 ? doc.getFieldValue(Const.ATH_PROP_SOURCE_JSON).toString()
944 : null;
945
946 String targetJson = doc.getFieldValue(Const.ATH_PROP_TARGET_JSON) != null
947 ? doc.getFieldValue(Const.ATH_PROP_TARGET_JSON).toString()
948 : null;
949
950 if (sourceJson != null) {
951 LayeredTextX slt = JacksonUtil.fromJson(sourceJson, LayeredTextX.class);
952 segment.setSource(slt);
953 }
954
955 if (targetJson != null) {
956 LayeredTextX tlt = JacksonUtil.fromJson(targetJson, LayeredTextX.class);
957 segment.setTarget(tlt);
958 }
959
960
961 segment.setCreatedAt(AthUtil.safeToDate(doc.get(Const.ATH_PROP_CREATED_AT), null));
962 segment.setUpdatedAt(AthUtil.safeToDate(doc.get(Const.ATH_PROP_UPDATED_AT), null));
963
964 segment.setCreatedBy(AthUtil.safeToUuid(
965 SolrUtil.safeGetField(doc, Const.ATH_PROP_CREATED_BY, null), null));
966
967 segment.setUpdatedBy(AthUtil.safeToUuid(
968 SolrUtil.safeGetField(doc, Const.ATH_PROP_UPDATED_BY, null), null));
969
970 return segment;
971
972 } catch (Exception e) {
973 Log.error(this.getClass(), e, "Error converting Solr document to TmSegment");
974 return null;
975 }
976 }
977
978 private SolrInputDocument toSolrDoc(TmSegment segment) throws AthException {
979 SolrInputDocument doc = new SolrInputDocument();
980
981 if (segment.getId() != null) {
982 doc.addField(Const.ATH_PROP_SOLR_ID, segment.getId());
983 }
984
985 if (segment.getTmSegId() != null) {
986 doc.addField(Const.ATH_PROP_TM_SEG_ID, segment.getTmSegId().toString());
987 }
988
989 if (segment.getTmId() != null) {
990 doc.addField(Const.ATH_PROP_TM_ID, segment.getTmId().toString());
991 }
992
993 if (segment.getTmFileName() != null) {
994 doc.addField(Const.ATH_PROP_TM_FILE_NAME, segment.getTmFileName());
995 }
996
997 if (segment.getDocId() != null) {
998 doc.addField(Const.ATH_PROP_DOC_ID, segment.getDocId().toString());
999 }
1000
1001 if (segment.getDocFileName() != null) {
1002 doc.addField(Const.ATH_PROP_DOC_FILE_NAME, segment.getDocFileName());
1003 }
1004
1005 if (segment.getSrcLang() != null) {
1006 doc.addField(Const.ATH_PROP_SRC_LANG, segment.getSrcLang());
1007 }
1008
1009 if (segment.getTrgLang() != null) {
1010 doc.addField(Const.ATH_PROP_TRG_LANG, segment.getTrgLang());
1011 }
1012
1013 ITextUnit tu = new TextUnit(segment.getId() != null ? segment.getId() : "temp");
1014
1015 if (segment.getSource() != null) {
1016 String sourceJson = JacksonUtil.toJson(segment.getSource(), false);
1017 SolrUtil.safeSetField(tu, doc, Const.ATH_PROP_SOURCE_JSON, sourceJson);
1018
1019 if (segment.getSource().getText() != null) {
1020 doc.addField(Const.ATH_PROP_SOURCE, segment.getSource().getText());
1021 }
1022
1023 if (segment.getSource().getTextWithCodes() != null) {
1024 SolrUtil.safeSetField(tu, doc, Const.ATH_PROP_SOURCE_WITH_CODES,
1025 segment.getSource().getTextWithCodes());
1026 }
1027 }
1028
1029 if (segment.getTarget() != null) {
1030 String targetJson = JacksonUtil.toJson(segment.getTarget(), false);
1031 SolrUtil.safeSetField(tu, doc, Const.ATH_PROP_TARGET_JSON, targetJson);
1032
1033 if (segment.getTarget().getText() != null) {
1034 doc.addField(Const.ATH_PROP_TARGET, segment.getTarget().getText());
1035 }
1036
1037 if (segment.getTarget().getTextWithCodes() != null) {
1038 SolrUtil.safeSetField(tu, doc, Const.ATH_PROP_TARGET_WITH_CODES,
1039 segment.getTarget().getTextWithCodes());
1040 }
1041 }
1042
1043 if (segment.getCreatedAt() != null) {
1044 doc.addField(Const.ATH_PROP_CREATED_AT, segment.getCreatedAt());
1045 }
1046
1047 if (segment.getUpdatedAt() != null) {
1048 doc.addField(Const.ATH_PROP_UPDATED_AT, segment.getCreatedAt());
1049 }
1050
1051 if (segment.getCreatedBy() != null) {
1052 doc.addField(Const.ATH_PROP_CREATED_BY, segment.getCreatedBy().toString());
1053 }
1054
1055 if (segment.getUpdatedBy() != null) {
1056 doc.addField(Const.ATH_PROP_UPDATED_BY, segment.getUpdatedBy().toString());
1057 }
1058
1059 return doc;
1060 }
1061 }