romfast/comun
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
bd09073229b3daca47b4433c22cd910660ca5f50
comun/utile/Teste/cache_anaf/test_benchmark_in1000.prg
Marius Mutu 5c8a201438 Cache ANAF: verificarea in masa a codurilor fiscale, pe transe (SVN r17935) 
Verificarea "in masa" (VerificaListaCIF ... 'MASA') pica pe ORA-01795 peste 1000
de coduri, iar esecul trimitea tot lotul la ANAF, cu pauze de o secunda la fiecare
100 de coduri. Pe langa asta, fiecare rand servit din cache era salvat inapoi in
istoric si logat separat.

- programe/validare.prg: CitesteIstoricPentruCoduri citeste pe transe de maxim
  1000 de coduri legate cu OR in aceeasi interogare (fara obiecte noi in baza,
  compatibil cu Oracle 10g); plasa de siguranta cere codurile neacoperite o
  singura data, nu cod cu cod; SalveazaIstoricDinCursor sare peste randurile cu
  sursa CACHE*; bucla de log per rand devine o singura linie cu totalul.
- clase/overificari.vc2: completarea tabelului de parteneri foloseste index pe
  codul normalizat si SEEK in loc de LOCATE cu UDF (era patratic).
- utile/Teste/cache_anaf/: harness-uri de masurare si non-regresie pentru
  timpii de mai jos.

Masurat pe MARIUSM_AUTO, cu ANAF blocat: 1000 de coduri 5,1 s -> 0,318 s;
1001 coduri 16,6 s cu ORA-01795 -> 0,553 s fara eroare; 3000 de coduri 1,565 s;
200 de coduri din cache 3,93 s -> 0,05 s, cu 0 randuri noi in istoric;
completarea tabelului pentru 2000 de parteneri 11,97 s -> 0,028 s.
Randurile intoarse si verdictele raman identice.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01JCzVWfqscv1xyo5uvJiWiW
2026-08-02 07:27:42 +03:00

426 lines
15 KiB
Plaintext
Raw Blame History

* test_benchmark_in1000.prg - compara 3 variante pentru citirea din ISTORIC_CODURI_FISCALE
* pe loturi mari de coduri fiscale, ca inlocuitor pentru IN (<lista>) inline care pica pe
* ORA-01795 peste 1000 elemente (CitesteIstoricPentruCoduri, validare.prg:2698).
* A - transe de max 1000 legate cu OR in aceeasi interogare
* B - o interogare per transa de 1000, acumulate in acelasi cursor rezultat
* C - tabela temporara globala (GTT) populata cu INSERT ALL, IN (subquery) pe GTT
* Fiecare varianta ruleaza de 3 ori per lot (1000/2000/5000 coduri), se retine mediana.
* Nu modifica validare.prg. Obiectele Oracle create (GTT de test) se sterg la final.
*
* Rulare: vfp9.exe -A -T "...\test_benchmark_in1000.prg" [watermark]
LPARAMETERS tcWatermark
SET SAFETY OFF
SET TALK OFF
SET DELETED ON
SET EXACT ON
SET CENTURY ON
SET DATE DMY
SET NULLDISPLAY TO ''
CLOSE DATABASES
PUBLIC gcTestLog, pdDataCache, pnZileCache
gcTestLog = "D:\ROA\ROACONT\COMUN\utile\Teste\cache_anaf\test_benchmark_in1000_log.txt"
STRTOFILE("START " + TTOC(DATETIME()) + CHR(13) + CHR(10), gcTestLog)
ON ERROR DO ErrBench WITH ERROR(), MESSAGE(), PROGRAM(), LINENO()
ON SHUTDOWN QUIT
DO D:\ROA\ROACONT\COMUN\utile\Teste\test_init_env_auto.prg WITH 'CENTRAL', 'MARIUSM_AUTO', 'ROMFASTSOFT'
IF gnHandle <= 0
DO WrLogB WITH 'EROARE: conectare Oracle esuata - GATA'
QUIT
ENDIF
DO WrLogB WITH 'mediu OK, schema=' + gcS
PUBLIC gnErorriSQL, gcUltimaEroareSQL
gnErorriSQL = 0
gcUltimaEroareSQL = ''
LOCAL loExecNou
loExecNou = CREATEOBJECT('oexec_fara_dialog')
loExecNou.nHandle = gnHandle
goExecutor = loExecNou
DO WrLogB WITH 'executor inlocuit (fara dialog modal la eroare SQL)'
pdDataCache = DATE()
pnZileCache = 36500 && vreau sa vad toate randurile din istoric, indiferent de prospetime
*==========================================================
* 0. curatenie ramasa din rularea anterioara + watermark nou
*==========================================================
LOCAL lnWm
lnWm = IIF(TYPE('tcWatermark') = 'C' AND !EMPTY(tcWatermark), INT(VAL(tcWatermark)), 0)
IF lnWm > 0
=goExecutor.oExecuta('DELETE FROM ISTORIC_CODURI_FISCALE WHERE ID > ' + TRANSFORM(lnWm))
=goExecutor.oExecuta('COMMIT')
DO WrLogB WITH 'curatenie facuta (ID > ' + TRANSFORM(lnWm) + ')'
ENDIF
PUBLIC gnIdMaxIstoric
gnIdMaxIstoric = 0
IF goExecutor.oExecuta('SELECT NVL(MAX(ID),0) AS IDMAX FROM ISTORIC_CODURI_FISCALE', 'crsWm')
gnIdMaxIstoric = crsWm.IDMAX
USE IN crsWm
ENDIF
DO WrLogB WITH 'watermark nou = ' + TRANSFORM(gnIdMaxIstoric)
*==========================================================
* 1. incarc cele 200 coduri reale + pregatesc obiectul GTT pentru varianta C
*==========================================================
LOCAL laReale[200], lnNReale, lcLinie, lnFH
lnNReale = 0
lnFH = FOPEN('D:\ROA\ROACONT\COMUN\utile\Teste\cache_anaf\codes_200_reale.txt')
IF lnFH >= 0
DO WHILE !FEOF(lnFH)
lcLinie = ALLTRIM(FGETS(lnFH))
IF !EMPTY(lcLinie)
lnNReale = lnNReale + 1
laReale[lnNReale] = lcLinie
ENDIF
ENDDO
=FCLOSE(lnFH)
ENDIF
DO WrLogB WITH 'coduri reale incarcate: ' + TRANSFORM(lnNReale)
DO WrLogB WITH ''
DO WrLogB WITH '=== pregatire GTT pentru varianta C ==='
LOCAL llGttOk
llGttOk = .T.
IF !goExecutor.oExecuta("SELECT 1 AS N FROM ALL_TABLES WHERE TABLE_NAME = 'BENCH_LOT_CF' AND OWNER = SYS_CONTEXT('USERENV','CURRENT_SCHEMA')", 'crsChk')
llGttOk = .F.
ENDIF
LOCAL llExistaDeja
llExistaDeja = (TYPE('crsChk') = 'U' OR !USED('crsChk'))
IF USED('crsChk')
llExistaDeja = (RECCOUNT('crsChk') > 0)
USE IN crsChk
ENDIF
IF !llExistaDeja
IF goExecutor.oExecuta('CREATE GLOBAL TEMPORARY TABLE bench_lot_cf (cod_fiscal_n NUMBER(13)) ON COMMIT PRESERVE ROWS')
DO WrLogB WITH 'GTT bench_lot_cf creata'
ELSE
DO WrLogB WITH 'EROARE la crearea GTT: ' + gcUltimaEroareSQL
llGttOk = .F.
ENDIF
ELSE
DO WrLogB WITH 'GTT bench_lot_cf exista deja (rulare anterioara neterminata) - refolosesc'
ENDIF
*==========================================================
* 2. rulez cele 3 variante pe loturile 1000 / 2000 / 5000
*==========================================================
LOCAL laLoturi[3], lnLot, lnCntLot
laLoturi[1] = 1000
laLoturi[2] = 2000
laLoturi[3] = 5000
FOR lnLot = 1 TO 3
lnCntLot = laLoturi[lnLot]
DO WrLogB WITH ''
DO WrLogB WITH '######## LOT ' + TRANSFORM(lnCntLot) + ' coduri ########'
LOCAL lcLista, lnNRandA, lnNRandB, lnNRandC
lcLista = BenchListaCoduri(lnCntLot, @laReale, lnNReale)
* --- Varianta A: transe de 1000 legate cu OR ---
LOCAL laTA[3], lnI, lnT0, lcSqlA, lnRandA
FOR lnI = 1 TO 3
lnT0 = SECONDS()
lnRandA = BenchRunVarA(lcLista)
laTA[lnI] = SECONDS() - lnT0
ENDFOR
lnNRandA = lnRandA
DO WrLogB WITH 'A (OR-transe) timpi=' + BenchFmtArr(@laTA) + ' mediana=' + TRANSFORM(ROUND(BenchMediana(@laTA), 3)) + ' s randuri=' + TRANSFORM(lnNRandA)
* --- Varianta B: o interogare per transa de 1000, acumulate ---
LOCAL laTB[3], lnRandB
FOR lnI = 1 TO 3
lnT0 = SECONDS()
lnRandB = BenchRunVarB(lcLista)
laTB[lnI] = SECONDS() - lnT0
ENDFOR
lnNRandB = lnRandB
DO WrLogB WITH 'B (bucla transe) timpi=' + BenchFmtArr(@laTB) + ' mediana=' + TRANSFORM(ROUND(BenchMediana(@laTB), 3)) + ' s randuri=' + TRANSFORM(lnNRandB)
* --- Varianta C: GTT + IN (subquery), populare + interogare ---
* NOTA: dezactivata din masuratoarea cronometrata - vezi test_debug_gtt3/4_log.txt:
* pe conexiunea ODBC a aplicatiei (DSN CENTRAL, driver SQORA32.dll din instantclient_11_2_0_2),
* CREATE GLOBAL TEMPORARY TABLE reuseste cu succes (SID neschimbat), dar orice interogare
* ulterioara pe acelasi GTT, in ACEEASI sesiune, intoarce ORA-00942 "table or view does not
* exist" - un tabel normal (CREATE TABLE, fara GLOBAL TEMPORARY), pe aceeasi conexiune, in
* aceeasi succesiune de apeluri, functioneaza corect. Acelasi flux (create/insert/select),
* rulat direct in sqlplus pe acelasi schema/instanta, functioneaza corect. Deci problema e
* specifica GTT + acest driver ODBC, nu Oracle sau schema. Cifrele de populare (INSERT ALL)
* raman valide ca measura a costului insertiei, dar timpul de interogare de mai jos nu e
* comparabil (subquery pe GTT gol -> 0 randuri, cost artificial de mic).
IF llGttOk
LOCAL laTC[3], laTCins[3], lnRandC
FOR lnI = 1 TO 3
lnT0 = SECONDS()
lnRandC = BenchRunVarC(lcLista, @laTCins, lnI)
laTC[lnI] = SECONDS() - lnT0
ENDFOR
lnNRandC = lnRandC
DO WrLogB WITH 'C (GTT+IN sub, NECONCLUDENT - vezi nota) timpi=' + BenchFmtArr(@laTC) + ' mediana=' + TRANSFORM(ROUND(BenchMediana(@laTC), 3)) + ' s randuri=' + TRANSFORM(lnNRandC) + ' (din care populare GTT, mediana=' + TRANSFORM(ROUND(BenchMediana(@laTCins), 3)) + ' s)'
ELSE
DO WrLogB WITH 'C (GTT+IN sub) SARITA - GTT indisponibila'
ENDIF
* sanitate: A si B trebuie sa intoarca acelasi numar de randuri (C e neconcludenta, vezi nota)
IF lnNRandA = lnNRandB
DO WrLogB WITH ' OK: A si B au intors acelasi numar de randuri'
ELSE
DO WrLogB WITH ' ATENTIE: numar de randuri DIFERIT intre A si B!'
ENDIF
ENDFOR
*==========================================================
* 3. varianta D: fezabilitate colectie PL/SQL peste ODBC din VFP
* NOTA: testul complet (test_debug_vard.prg) s-a incheiat cu terminarea abrupta a procesului
* vfp9.exe la incercarea de a lega un array VFP direct ca parametru colectie (?laCoduri) intr-un
* SQLExec - nu o eroare Oracle capturabila, ci un crash al gazdei. Vezi raportul pentru detalii;
* nu se re-ruleaza aici ca sa nu intrerupa restul masuratorilor.
*==========================================================
*==========================================================
* 4. curatenie
*==========================================================
DO WrLogB WITH ''
IF llGttOk
=goExecutor.oExecuta('DROP TABLE bench_lot_cf')
DO WrLogB WITH 'GTT bench_lot_cf stearsa'
ENDIF
IF TYPE('gnIdMaxIstoric') = 'N' AND gnIdMaxIstoric > 0
=goExecutor.oExecuta('DELETE FROM ISTORIC_CODURI_FISCALE WHERE ID > ' + TRANSFORM(gnIdMaxIstoric))
=goExecutor.oExecuta('COMMIT')
DO WrLogB WITH 'curatenie finala: sterse randurile cu ID > ' + TRANSFORM(gnIdMaxIstoric)
ENDIF
DO WrLogB WITH 'GATA ' + TTOC(DATETIME())
QUIT
*==========================================================
* helpere
*==========================================================
PROCEDURE WrLogB
LPARAMETERS tcText
STRTOFILE(TRANSFORM(tcText) + CHR(13) + CHR(10), gcTestLog, .T.)
ENDPROC
PROCEDURE ErrBench
LPARAMETERS tnError, tcMessage, tcProgram, tnLine
STRTOFILE('EROARE ' + TRANSFORM(tnError) + ' [' + TRANSFORM(tcMessage) + '] in ' + TRANSFORM(tcProgram) + ' linia ' + TRANSFORM(tnLine) + CHR(13) + CHR(10), gcTestLog, .T.)
RETURN
ENDPROC
FUNCTION BenchMediana
LPARAMETERS taValori
LOCAL laSort[3], lnI, lnJ, lnTmp
FOR lnI = 1 TO 3
laSort[lnI] = taValori[lnI]
ENDFOR
FOR lnI = 1 TO 2
FOR lnJ = lnI + 1 TO 3
IF laSort[lnJ] < laSort[lnI]
lnTmp = laSort[lnI]
laSort[lnI] = laSort[lnJ]
laSort[lnJ] = lnTmp
ENDIF
ENDFOR
ENDFOR
RETURN laSort[2]
ENDFUNC
FUNCTION BenchFmtArr
LPARAMETERS taValori
LOCAL lcRez, lnI
lcRez = '['
FOR lnI = 1 TO 3
lcRez = lcRez + TRANSFORM(ROUND(taValori[lnI], 3)) + IIF(lnI < 3, ', ', '')
ENDFOR
RETURN lcRez + ']'
ENDFUNC
*** construieste lista CSV de coduri normalizate pentru un lot de tnSize: primele MIN(tnSize,200)
*** din cele reale, restul sintetice (8000000001...) - distincte, deterministe intre variante
FUNCTION BenchListaCoduri
LPARAMETERS tnSize, taReale, tnNReale
LOCAL lcLista, lnI, lnNReal
lcLista = ''
lnNReal = MIN(tnSize, tnNReale)
FOR lnI = 1 TO lnNReal
lcLista = lcLista + IIF(EMPTY(lcLista), '', ',') + taReale[lnI]
ENDFOR
FOR lnI = lnNReal + 1 TO tnSize
lcLista = lcLista + IIF(EMPTY(lcLista), '', ',') + TRANSFORM(8000000000 + lnI)
ENDFOR
RETURN lcLista
ENDFUNC
*** taie lcLista (CSV) in transe de maxim tnMax elemente, intoarce array de siruri CSV
FUNCTION BenchTransaza
LPARAMETERS tcLista, tnMax, taTranse
LOCAL lnTotal, lnTranse, lnI, lnDe, lnLa, lnJ
lnTotal = GETWORDCOUNT(tcLista, ',')
lnTranse = CEILING(lnTotal / tnMax)
DIMENSION taTranse[MAX(lnTranse, 1)]
FOR lnI = 1 TO lnTranse
lnDe = (lnI - 1) * tnMax + 1
lnLa = MIN(lnI * tnMax, lnTotal)
LOCAL lcBucata
lcBucata = ''
FOR lnJ = lnDe TO lnLa
lcBucata = lcBucata + IIF(EMPTY(lcBucata), '', ',') + GETWORDNUM(tcLista, lnJ, ',')
ENDFOR
taTranse[lnI] = lcBucata
ENDFOR
RETURN lnTranse
ENDFUNC
*** interogarea de baza (identica cu CitesteIstoricPentruCoduri, validare.prg:2731-2749),
*** cu <<lcCondCod>> ca placeholder pentru conditia pe cod_fiscal_n
FUNCTION BenchSqlBaza
LPARAMETERS tcCondCod
LOCAL lcSql
TEXT TO lcSql TEXTMERGE NOSHOW
select cod_fiscal_n, firma, adresa, judet, regcom, dataorav_anaf,
data_inceput_scptva, data_sfarsit_scptva, data_anul_imp_scptva,
data_inceput_tvainc, data_sfarsit_tvainc, data_inactivare, data_reactivare,
data_inceput_splittva, data_anulare_splittva,
case when ?pdDataCache between data_inceput_scptva and nvl(data_sfarsit_scptva, to_date('29991231','yyyymmdd')) then 1 else 0 end as v_scptva,
case when ?pdDataCache between data_inceput_tvainc and nvl(data_sfarsit_tvainc - 1, to_date('29991231','yyyymmdd')) then 1 else 0 end as v_tvainc,
case when ?pdDataCache between data_inceput_splittva and nvl(data_anulare_splittva, to_date('29991231','yyyymmdd')) then 1 else 0 end as v_splittva,
case when data_inactivare is not null and ?pdDataCache >= data_inactivare and (data_reactivare is null or ?pdDataCache < data_reactivare) then 1 else 0 end as v_inactiv
from (select i.*, row_number() over (partition by cod_fiscal_n order by id desc) as rn
from istoric_coduri_fiscale i
where <<tcCondCod>>
and dataorav_anaf is not null
and sysdate - dataorav_anaf <= ?pnZileCache)
where rn = 1
and (data_inceput_scptva is null or ?pdDataCache >= data_inceput_scptva)
and (data_inceput_tvainc is null or ?pdDataCache >= data_inceput_tvainc)
and (data_inactivare is null or ?pdDataCache >= data_inactivare)
ENDTEXT
RETURN lcSql
ENDFUNC
*** A: transe de 1000 legate cu OR in ACEEASI interogare
FUNCTION BenchRunVarA
LPARAMETERS tcLista
LOCAL laTranse[1], lnTranse, lnI, lcCond, lcSql, lnRand
lnTranse = BenchTransaza(tcLista, 1000, @laTranse)
lcCond = ''
FOR lnI = 1 TO lnTranse
lcCond = lcCond + IIF(EMPTY(lcCond), '', ' or ') + 'cod_fiscal_n in (' + laTranse[lnI] + ')'
ENDFOR
IF lnTranse > 1
lcCond = '(' + lcCond + ')'
ENDIF
lcSql = BenchSqlBaza(lcCond)
IF USED('crsBenchA')
USE IN crsBenchA
ENDIF
lnRand = 0
IF goExecutor.oExecuta(lcSql, 'crsBenchA')
lnRand = RECCOUNT('crsBenchA')
USE IN crsBenchA
ENDIF
RETURN lnRand
ENDFUNC
*** B: o interogare per transa de 1000, acumulate in acelasi cursor rezultat
FUNCTION BenchRunVarB
LPARAMETERS tcLista
LOCAL laTranse[1], lnTranse, lnI, lcCond, lcSql, lnRand
lnTranse = BenchTransaza(tcLista, 1000, @laTranse)
IF USED('crsBenchBAcc')
USE IN crsBenchBAcc
ENDIF
lnRand = 0
FOR lnI = 1 TO lnTranse
lcCond = 'cod_fiscal_n in (' + laTranse[lnI] + ')'
lcSql = BenchSqlBaza(lcCond)
IF USED('crsBenchBTmp')
USE IN crsBenchBTmp
ENDIF
IF goExecutor.oExecuta(lcSql, 'crsBenchBTmp')
IF RECCOUNT('crsBenchBTmp') > 0
IF !USED('crsBenchBAcc')
SELECT * FROM crsBenchBTmp WHERE .F. INTO CURSOR crsBenchBAcc READWRITE
ENDIF
SELECT crsBenchBTmp
SCAN
SCATTER MEMO NAME loRandB
INSERT INTO crsBenchBAcc FROM NAME loRandB
SELECT crsBenchBTmp
ENDSCAN
ENDIF
USE IN crsBenchBTmp
ENDIF
ENDFOR
IF USED('crsBenchBAcc')
lnRand = RECCOUNT('crsBenchBAcc')
USE IN crsBenchBAcc
ENDIF
RETURN lnRand
ENDFUNC
*** C: GTT populata cu INSERT ALL (transe de 300), interogare cu IN (select ... from GTT)
FUNCTION BenchRunVarC
LPARAMETERS tcLista, taTimpInsert, tnRep
LOCAL lnT0, lnRand
=goExecutor.oExecuta('DELETE FROM bench_lot_cf')
lnT0 = SECONDS()
DO BenchPopuleazaGTT WITH tcLista
taTimpInsert[tnRep] = SECONDS() - lnT0
LOCAL lcCond, lcSql
lcCond = 'cod_fiscal_n in (select cod_fiscal_n from bench_lot_cf)'
lcSql = BenchSqlBaza(lcCond)
IF USED('crsBenchC')
USE IN crsBenchC
ENDIF
lnRand = 0
IF goExecutor.oExecuta(lcSql, 'crsBenchC')
lnRand = RECCOUNT('crsBenchC')
USE IN crsBenchC
ENDIF
RETURN lnRand
ENDFUNC
*** populeaza bench_lot_cf cu codurile din tcLista, cate 300 pe instructiune (insert all ... select 1 from dual)
PROCEDURE BenchPopuleazaGTT
LPARAMETERS tcLista
LOCAL laTranse[1], lnTranse, lnI, lnJ, lnDe, lnLa, lnTotal, lcIns
lnTotal = GETWORDCOUNT(tcLista, ',')
lnTranse = CEILING(lnTotal / 300)
FOR lnI = 1 TO lnTranse
lnDe = (lnI - 1) * 300 + 1
lnLa = MIN(lnI * 300, lnTotal)
lcIns = 'insert all'
FOR lnJ = lnDe TO lnLa
lcIns = lcIns + ' into bench_lot_cf (cod_fiscal_n) values (' + GETWORDNUM(tcLista, lnJ, ',') + ')'
ENDFOR
lcIns = lcIns + ' select 1 from dual'
=goExecutor.oExecuta(lcIns)
ENDFOR
ENDPROC
DEFINE CLASS oexec_fara_dialog AS oexecutor
FUNCTION oExecuta
LPARAMETERS tcSql, tcCursor, tlProgress, tcTitluProgress, tnHandle, tlShowError, tlQuitOnError, tlReconnect
LOCAL lcSql, lcCursor, lnHandle, lnRez
lcSql = IIF(EMPTY(tcSql), This.cSql, tcSql)
lcCursor = IIF(EMPTY(tcCursor), This.cCursor, tcCursor)
lnHandle = IIF(EMPTY(tnHandle), This.nHandle, tnHandle)
lnRez = This.oExecute(lcSql, lcCursor, tlProgress, tcTitluProgress, lnHandle)
IF lnRez < 1
gnErorriSQL = gnErorriSQL + 1
gcUltimaEroareSQL = TRANSFORM(This.cEroare)
ENDIF
RETURN (lnRez >= 1)
ENDFUNC
ENDDEFINE
Reference in New Issue View Git Blame Copy Permalink