Backup: JSON optimization attempt with direct string merge

This version attempted to optimize JSON processing by:
- Direct string concatenation without deserialization
- MergeProductsJsonDirect and MergeOrdersJsonDirect functions
- Direct STRTOFILE instead of nfJsonCreate

Issues found:
- Only 100 products saved instead of 812 (merge failed)
- Invalid JSON syntax in orders: 'discounts':[,
- Pattern matching errors for GoMag API structure

Reverting to stable version 8324a26 next.

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
2025-08-27 17:22:05 +03:00
parent fd48ca480b
commit 6f362f41e7
2 changed files with 458 additions and 107 deletions

View File

@@ -28,9 +28,10 @@ SET DEFAULT TO (m.gcAppPath)
lcPath = gcAppPath + 'nfjson;'
SET PATH TO (m.lcPath) ADDITIVE
SET PROCEDURE TO nfjsonread.prg ADDITIVE
SET PROCEDURE TO utils.prg ADDITIVE
SET PROCEDURE TO nfjsonread.prg ADDITIVE
SET PROCEDURE TO nfjsoncreate.prg ADDITIVE
SET PROCEDURE TO regex.prg ADDITIVE
*-- Initializare logging si statistici
gnStartTime = SECONDS()
@@ -108,11 +109,9 @@ ENDTRY
IF llGetProducts
LogMessage("[PRODUCTS] Starting product retrieval", "INFO", gcLogFile)
*-- Bucla pentru preluarea tuturor produselor (paginare)
loAllJsonData = CREATEOBJECT("Empty")
ADDPROPERTY(loAllJsonData, "products", CREATEOBJECT("Empty"))
ADDPROPERTY(loAllJsonData, "total", 0)
ADDPROPERTY(loAllJsonData, "pages", 0)
*-- Bucla pentru preluarea tuturor produselor (paginare) - optimizare JSON direct
LOCAL lcAllProductsJson
lcAllProductsJson = ""
lnTotalProducts = 0
DO WHILE llHasMorePages
@@ -147,28 +146,31 @@ IF llGetProducts
*-- Success - preluare raspuns
lcResponse = loHttp.ResponseText
*-- Parsare JSON cu nfjson
SET PATH TO nfjson ADDITIVE
loJsonData = nfJsonRead(lcResponse)
IF !ISNULL(loJsonData)
*-- Prima pagina - setam informatiile generale
IF lnCurrentPage = 1
*-- Optimizare: folosim JSON direct fara parsare completa
IF lnCurrentPage = 1
*-- Prima pagina - parsam doar pentru metadata
SET PATH TO nfjson ADDITIVE
loJsonData = nfJsonRead(lcResponse)
IF !ISNULL(loJsonData)
IF TYPE('loJsonData.total') = 'C' OR TYPE('loJsonData.total') = 'N'
loAllJsonData.total = VAL(TRANSFORM(loJsonData.total))
lnTotalProducts = VAL(TRANSFORM(loJsonData.total))
ENDIF
IF TYPE('loJsonData.pages') = 'C' OR TYPE('loJsonData.pages') = 'N'
loAllJsonData.pages = VAL(TRANSFORM(loJsonData.pages))
lnTotalPages = VAL(TRANSFORM(loJsonData.pages))
ENDIF
LogMessage("[PRODUCTS] Total items: " + TRANSFORM(loAllJsonData.total) + " | Pages: " + TRANSFORM(loAllJsonData.pages), "INFO", gcLogFile)
LogMessage("[PRODUCTS] Total items: " + TRANSFORM(lnTotalProducts) + " | Pages: " + TRANSFORM(lnTotalPages), "INFO", gcLogFile)
ENDIF
*-- Adaugare produse din pagina curenta
IF TYPE('loJsonData.products') = 'O'
DO MergeProducts WITH loAllJsonData, loJsonData
ENDIF
*-- Salvam JSON-ul complet pentru prima pagina
lcAllProductsJson = lcResponse
ELSE
*-- Paginile urmatoare - merge direct JSON
lcAllProductsJson = MergeProductsJsonDirect(lcAllProductsJson, lcResponse)
ENDIF
*-- Verificare daca mai sunt pagini
*-- Verificare daca mai sunt pagini
IF lnCurrentPage = 1 AND !ISNULL(loJsonData)
IF TYPE('loJsonData.pages') = 'C' OR TYPE('loJsonData.pages') = 'N'
lnTotalPages = VAL(TRANSFORM(loJsonData.pages))
IF lnCurrentPage >= lnTotalPages
@@ -180,14 +182,14 @@ IF llGetProducts
llHasMorePages = .F.
ENDIF
ENDIF
lnCurrentPage = lnCurrentPage + 1
ELSE
*-- Eroare parsare JSON
LogMessage("[PRODUCTS] ERROR: JSON parsing failed for page " + TRANSFORM(lnCurrentPage), "ERROR", gcLogFile)
llHasMorePages = .F.
*-- Pentru paginile urmatoare, verificam daca am ajuns la limita
IF lnTotalPages > 0 AND lnCurrentPage >= lnTotalPages
llHasMorePages = .F.
ENDIF
ENDIF
lnCurrentPage = lnCurrentPage + 1
ELSE
*-- Eroare HTTP
@@ -219,16 +221,12 @@ IF llGetProducts
ENDDO
*-- Salvare array JSON cu toate produsele
IF !ISNULL(loAllJsonData) AND TYPE('loAllJsonData.products') = 'O'
*-- Salvare JSON direct (optimizat)
IF !EMPTY(lcAllProductsJson)
lcJsonFileName = lcOutputDir + "\gomag_all_products_" + DTOS(DATE()) + "_" + STRTRAN(TIME(), ":", "") + ".json"
DO SaveProductsArray WITH loAllJsonData, lcJsonFileName
STRTOFILE(lcAllProductsJson, lcJsonFileName)
LogMessage("[PRODUCTS] JSON saved: " + lcJsonFileName, "INFO", gcLogFile)
*-- Calculam numarul de produse procesate
IF TYPE('loAllJsonData.products') = 'O'
lnPropCount = AMEMBERS(laProducts, loAllJsonData.products, 0)
gnProductsProcessed = lnPropCount
ENDIF
gnProductsProcessed = lnTotalProducts
ENDIF
ELSE
@@ -239,13 +237,11 @@ ENDIF
IF llGetOrders
LogMessage("[ORDERS] Starting orders retrieval (last " + TRANSFORM(loSettings.OrderDaysBack) + " days from " + lcStartDateStr + ")", "INFO", gcLogFile)
*-- Reinitializare pentru comenzi
*-- Reinitializare pentru comenzi - optimizare JSON direct
lnCurrentPage = 1
llHasMorePages = .T.
loAllOrderData = CREATEOBJECT("Empty")
ADDPROPERTY(loAllOrderData, "orders", CREATEOBJECT("Empty"))
ADDPROPERTY(loAllOrderData, "total", 0)
ADDPROPERTY(loAllOrderData, "pages", 0)
LOCAL lcAllOrdersJson
lcAllOrdersJson = ""
*-- Bucla pentru preluarea comenzilor
DO WHILE llHasMorePages
@@ -280,93 +276,63 @@ DO WHILE llHasMorePages
*-- Success - preluare raspuns
lcResponse = loHttp.ResponseText
*-- Parsare JSON cu nfjson
SET PATH TO nfjson ADDITIVE
loJsonData = nfJsonRead(lcResponse)
IF !ISNULL(loJsonData)
*-- Debug: Afisam structura JSON pentru prima pagina
IF lnCurrentPage = 1
*-- Optimizare: folosim JSON direct pentru comenzi
IF lnCurrentPage = 1
*-- Prima pagina - parsam doar pentru metadata si salvam JSON-ul complet
loJsonData = nfJsonRead(lcResponse)
IF !ISNULL(loJsonData)
LogMessage("[ORDERS] DEBUG: Analyzing JSON structure...", "DEBUG", gcLogFile)
lnPropCount = AMEMBERS(laJsonProps, loJsonData, 0)
FOR lnDebugIndex = 1 TO MIN(lnPropCount, 10) && Primele 10 proprietati
FOR lnDebugIndex = 1 TO MIN(lnPropCount, 10)
lcPropName = laJsonProps(lnDebugIndex)
lcPropType = TYPE('loJsonData.' + lcPropName)
LogMessage("[ORDERS] Property: " + lcPropName + " (Type: " + lcPropType + ")", "DEBUG", gcLogFile)
ENDFOR
ENDIF
*-- Prima pagina - setam informatiile generale
IF lnCurrentPage = 1
LOCAL lnTotalOrders
IF TYPE('loJsonData.total') = 'C' OR TYPE('loJsonData.total') = 'N'
loAllOrderData.total = VAL(TRANSFORM(loJsonData.total))
lnTotalOrders = VAL(TRANSFORM(loJsonData.total))
ENDIF
IF TYPE('loJsonData.pages') = 'C' OR TYPE('loJsonData.pages') = 'N'
loAllOrderData.pages = VAL(TRANSFORM(loJsonData.pages))
lnTotalPages = VAL(TRANSFORM(loJsonData.pages))
ENDIF
LogMessage("[ORDERS] Total items: " + TRANSFORM(loAllOrderData.total) + " | Pages: " + TRANSFORM(loAllOrderData.pages), "INFO", gcLogFile)
ENDIF
*-- Adaugare comenzi din pagina curenta
*-- API-ul GoMag returneaza un array direct de comenzi
LOCAL llHasOrders, lnOrdersFound
llHasOrders = .F.
lnOrdersFound = 0
*-- Verificam daca JSON-ul contine proprietatea orders
IF TYPE('loJsonData.orders') = 'O'
*-- Numaram comenzile din obiectul orders
LOCAL ARRAY laOrdersProps[1]
LOCAL lnOrdersCount
lnOrdersCount = AMEMBERS(laOrdersProps, loJsonData.orders, 0)
LogMessage("[ORDERS] Total items: " + TRANSFORM(lnTotalOrders) + " | Pages: " + TRANSFORM(lnTotalPages), "INFO", gcLogFile)
IF lnOrdersCount > 0
DO MergeOrdersArray WITH loAllOrderData, loJsonData
llHasOrders = .T.
lnOrdersFound = lnOrdersCount
LogMessage("[ORDERS] Found " + TRANSFORM(lnOrdersCount) + " orders in page " + TRANSFORM(lnCurrentPage), "INFO", gcLogFile)
gnOrdersProcessed = gnOrdersProcessed + lnOrdersCount
ENDIF
ELSE
*-- JSON-ul este direct un array de comenzi (backup logic)
lnDirectProps = AMEMBERS(laDirectProps, loJsonData, 0)
IF lnDirectProps > 0
*-- Cream un obiect temporar cu structura asteptata
LOCAL loTempData
loTempData = CREATEOBJECT("Empty")
ADDPROPERTY(loTempData, "orders", loJsonData)
DO MergeOrdersArray WITH loAllOrderData, loTempData
llHasOrders = .T.
lnOrdersFound = lnDirectProps
LogMessage("[ORDERS] Found " + TRANSFORM(lnDirectProps) + " orders in page " + TRANSFORM(lnCurrentPage), "INFO", gcLogFile)
gnOrdersProcessed = gnOrdersProcessed + lnDirectProps
*-- Calculam comenzile din prima pagina
LOCAL lnFirstPageOrders
lnFirstPageOrders = 0
IF TYPE('loJsonData.orders') = 'O'
lnFirstPageOrders = AMEMBERS(laTemp, loJsonData.orders, 0)
ENDIF
gnOrdersProcessed = gnOrdersProcessed + lnFirstPageOrders
ENDIF
IF !llHasOrders
LogMessage("[ORDERS] WARNING: No orders found in JSON response for page " + TRANSFORM(lnCurrentPage), "WARN", gcLogFile)
ENDIF
*-- Salvam JSON-ul complet pentru prima pagina
lcAllOrdersJson = lcResponse
ELSE
*-- Paginile urmatoare - merge direct JSON
lcAllOrdersJson = MergeOrdersJsonDirect(lcAllOrdersJson, lcResponse)
*-- Estimare comenzi procesate din dimensiunea JSON-ului
gnOrdersProcessed = gnOrdersProcessed + INT(LEN(lcResponse) / 500)
ENDIF
*-- Verificare daca mai sunt pagini
*-- Verificare daca mai sunt pagini
IF lnCurrentPage = 1 AND !ISNULL(loJsonData)
IF TYPE('loJsonData.pages') = 'C' OR TYPE('loJsonData.pages') = 'N'
lnTotalPages = VAL(TRANSFORM(loJsonData.pages))
IF lnCurrentPage >= lnTotalPages
llHasMorePages = .F.
ENDIF
ELSE
*-- Daca nu avem info despre pagini, verificam daca sunt comenzi
IF !llHasOrders
llHasMorePages = .F.
ENDIF
ENDIF
lnCurrentPage = lnCurrentPage + 1
ELSE
*-- Eroare parsare JSON pentru comenzi
LogMessage("[ORDERS] ERROR: JSON parsing failed for page " + TRANSFORM(lnCurrentPage), "ERROR", gcLogFile)
llHasMorePages = .F.
*-- Pentru paginile urmatoare, verificam daca am ajuns la limita
IF lnTotalPages > 0 AND lnCurrentPage >= lnTotalPages
llHasMorePages = .F.
ENDIF
ENDIF
lnCurrentPage = lnCurrentPage + 1
ELSE
*-- Eroare HTTP pentru comenzi
@@ -398,10 +364,10 @@ DO WHILE llHasMorePages
ENDDO
*-- Salvare array JSON cu toate comenzile
IF !ISNULL(loAllOrderData) AND TYPE('loAllOrderData.orders') = 'O'
*-- Salvare JSON direct (optimizat)
IF !EMPTY(lcAllOrdersJson)
lcOrderJsonFileName = lcOutputDir + "\gomag_orders_last7days_" + DTOS(DATE()) + "_" + STRTRAN(TIME(), ":", "") + ".json"
DO SaveOrdersArray WITH loAllOrderData, lcOrderJsonFileName
STRTOFILE(lcAllOrdersJson, lcOrderJsonFileName)
LogMessage("[ORDERS] JSON saved: " + lcOrderJsonFileName, "INFO", gcLogFile)
ENDIF
@@ -549,6 +515,123 @@ ENDIF
ENDPROC
*-- Functie optimizata pentru merge direct JSON produse (fara deserializare)
PROCEDURE MergeProductsJsonDirect
PARAMETERS tcFirstPageJson, tcNextPageJson
LOCAL lcEndPos, lcStart, lcEnd, lcNewProperties, lcResult
*-- Gaseste sfârșitul secțiunii products din prima pagina - cautam pozitia } inainte de "total"
lcEndPos = AT('},\"total\"', tcFirstPageJson)
IF lcEndPos = 0
*-- Fallback: gaseste ultimul } inainte de total
lcEndPos = AT('\"total\"', tcFirstPageJson)
IF lcEndPos > 0
lcEndPos = RAT('}', LEFT(tcFirstPageJson, lcEndPos - 1))
ENDIF
ENDIF
IF lcEndPos = 0
*-- Nu putem face merge, returnam prima pagina
RETURN tcFirstPageJson
ENDIF
*-- Debug logging pentru debugging
LogMessage("[PRODUCTS] DEBUG: Starting merge - Page 1 size: " + TRANSFORM(LEN(tcFirstPageJson)) + " | Next page size: " + TRANSFORM(LEN(tcNextPageJson)), "DEBUG", gcLogFile)
LogMessage("[PRODUCTS] DEBUG: Products end position found at: " + TRANSFORM(lcEndPos), "DEBUG", gcLogFile)
*-- Daca nu putem extrage cu regex, incercam metoda manuala
lcStart = AT('\"products\":{', tcNextPageJson)
IF lcStart > 0
lcStart = lcStart + 11 && dupa {"products":{
lcEnd = AT('},\"total\"', tcNextPageJson)
IF lcEnd = 0
lcEnd = AT('\"total\"', tcNextPageJson)
IF lcEnd > 0
lcEnd = RAT('}', LEFT(tcNextPageJson, lcEnd - 1))
ENDIF
ENDIF
IF lcEnd > lcStart
lcNewProperties = SUBSTR(tcNextPageJson, lcStart, lcEnd - lcStart)
LogMessage("[PRODUCTS] DEBUG: Extracted " + TRANSFORM(LEN(lcNewProperties)) + " chars from next page", "DEBUG", gcLogFile)
*-- Insereaza proprietatile cu virgula separator
lcResult = STUFF(tcFirstPageJson, lcEndPos, 0, ',' + lcNewProperties)
LogMessage("[PRODUCTS] DEBUG: Merge successful, result size: " + TRANSFORM(LEN(lcResult)), "DEBUG", gcLogFile)
RETURN lcResult
ELSE
LogMessage("[PRODUCTS] ERROR: Invalid extraction positions - Start: " + TRANSFORM(lcStart) + ", End: " + TRANSFORM(lcEnd), "ERROR", gcLogFile)
ENDIF
ENDIF
*-- Daca nu putem extrage, returnam prima pagina
RETURN tcFirstPageJson
ENDPROC
*-- Functie optimizata pentru merge direct JSON comenzi (fara deserializare)
PROCEDURE MergeOrdersJsonDirect
PARAMETERS tcFirstPageJson, tcNextPageJson
LOCAL lcEndPos, lcStart, lcEnd, lcNewOrders, lcResult
LOCAL lcTotalPos
*-- Debug logging pentru debugging
LogMessage("[ORDERS] DEBUG: Starting merge - Page 1 size: " + TRANSFORM(LEN(tcFirstPageJson)) + " | Next page size: " + TRANSFORM(LEN(tcNextPageJson)), "DEBUG", gcLogFile)
*-- Cautam pozitia pentru inserare inainte de "total", "page" sau "pages"
lcEndPos = AT('},\"total\"', tcFirstPageJson)
IF lcEndPos = 0
lcEndPos = AT('},\"page\"', tcFirstPageJson)
ENDIF
IF lcEndPos = 0
lcEndPos = AT('},\"pages\"', tcFirstPageJson)
ENDIF
IF lcEndPos = 0
LogMessage("[ORDERS] ERROR: Cannot find orders end position in first page", "ERROR", gcLogFile)
RETURN tcFirstPageJson
ENDIF
LogMessage("[ORDERS] DEBUG: Orders end position found at: " + TRANSFORM(lcEndPos), "DEBUG", gcLogFile)
*-- Extrage doar continutul din "orders":{...} din pagina urmatoare
lcStart = AT('\"orders\":{', tcNextPageJson)
IF lcStart > 0
lcStart = lcStart + 10 && dupa {"orders":{
*-- Gaseste sfarsitul obiectului orders din pagina urmatoare
lcEnd = AT('},\"total\"', tcNextPageJson)
IF lcEnd = 0
lcEnd = AT('},\"page\"', tcNextPageJson)
ENDIF
IF lcEnd = 0
lcEnd = AT('},\"pages\"', tcNextPageJson)
ENDIF
IF lcEnd > lcStart
lcNewOrders = SUBSTR(tcNextPageJson, lcStart, lcEnd - lcStart)
LogMessage("[ORDERS] DEBUG: Extracted " + TRANSFORM(LEN(lcNewOrders)) + " chars from next page", "DEBUG", gcLogFile)
*-- Curatam orice } de la final daca exista
lcNewOrders = RTRIM(lcNewOrders, ' }')
*-- Insereaza noile comenzi cu virgula
lcResult = STUFF(tcFirstPageJson, lcEndPos, 0, ',' + lcNewOrders)
LogMessage("[ORDERS] DEBUG: Merge successful, result size: " + TRANSFORM(LEN(lcResult)), "DEBUG", gcLogFile)
RETURN lcResult
ELSE
LogMessage("[ORDERS] ERROR: Invalid extraction positions - Start: " + TRANSFORM(lcStart) + ", End: " + TRANSFORM(lcEnd), "ERROR", gcLogFile)
ENDIF
ENDIF
*-- Daca nu putem extrage, returnam prima pagina
RETURN tcFirstPageJson
ENDPROC
*-- Functiile utilitare au fost mutate in utils.prg
*-- Scriptul cu paginare completa pentru preluarea tuturor produselor si comenzilor