--The object type CREATE OR REPLACE TYPE "TY_TERMINE" AS OBJECT( datum date, bezeichnung varchar2(1999), zeitbezug varchar2(1999), eflow varchar2(1), email varchar2(500), frist_id number, frist_sperre_flag varchar2(1), farbe varchar2(200) ) / --The pipelined function for dynamic row creation CREATE OR REPLACE FUNCTION SK20110621_TEST RETURN T_TERMINE PIPELINED IS BEGIN FOR i IN 1..1000000 LOOP PIPE ROW(ty_termine( SYSDATE - i , i||'. Offset' , 'Time ref.' , 'N' , 'sk@somewhere.de' , i , 'Y' , 'Green' )); END LOOP; END; / --The view for the pipelined function call (entity candidate) CREATE OR REPLACE view SK20110621_V AS SELECT * FROM TABLE(SK20110621_TEST) / --To be complete for DML: the insteadof trigger CREATE TRIGGER SK20110621_TRG INSTEAD OF UPDATE OR INSERT OR DELETE ON SK20110621_V BEGIN NULL; END; / ------------------------------------------------------------------------- package test; import java.util.Date; import javax.persistence.Entity; import javax.persistence.Id; import javax.persistence.Table; import javax.persistence.Temporal; import javax.persistence.TemporalType; @Entity(name="MyEntity") //@ReadOnly @Table(name="SK20110621_V") public class MyEntity{ @Temporal(TemporalType.DATE) private Date datum; private String bezeichnung; private String zeitbezug; private String eflow; private String email; @Id private int frist_id; private String frist_sperre_flag; private String farbe; public Date getDatum() { return datum; } public void setDatum(Date datum) { this.datum = datum; } public String getBezeichnung() { return bezeichnung; } public void setBezeichnung(String bezeichnung) { this.bezeichnung = bezeichnung; } public String getZeitbezug() { return zeitbezug; } public void setZeitbezug(String zeitbezug) { this.zeitbezug = zeitbezug; } public String getEflow() { return eflow; } public void setEflow(String eflow) { this.eflow = eflow; } public String getEmail() { return email; } public void setEmail(String email) { this.email = email; } public int getFrist_id() { return frist_id; } public void setFrist_id(int frist_id) { this.frist_id = frist_id; } public String getFrist_sperre_flag() { return frist_sperre_flag; } public void setFrist_sperre_flag(String frist_sperre_flag) { this.frist_sperre_flag = frist_sperre_flag; } public String getFarbe() { return farbe; } public void setFarbe(String farbe) { this.farbe = farbe; } } ------------------------------------------------------------------------- The main - Class "Runner" ------------------------------------------------------------------------- package test; import java.util.ArrayList; import java.util.Calendar; import java.util.Date; import java.util.GregorianCalendar; import java.util.Iterator; import java.util.List; //import java.util.Map; import javax.persistence.EntityManager; import javax.persistence.EntityManagerFactory; import javax.persistence.Persistence; import javax.persistence.Query; import org.eclipse.persistence.config.HintValues; import org.eclipse.persistence.config.QueryHints; public class Runner { /** * @param args */ private static void log(String log){ GregorianCalendar gc = new GregorianCalendar(); gc.setTimeInMillis(System.currentTimeMillis()); System.out.println(gc.getTime()+" "+gc.get(Calendar.MILLISECOND)+" "+log+"["+Runtime.getRuntime().totalMemory()/1024/1024+"]"); } private static void gc(int j){ Object o = new Object(); for (int i = 0; i < j; i++) { synchronized( o ) { try { o.wait(1000); } catch ( InterruptedException e ) { } } log("Wait "+i+" done"); System.gc(); } } private static void em(){ EntityManagerFactory factory = Persistence.createEntityManagerFactory("myPU"); EntityManager em = factory.createEntityManager(); Query q = em.createQuery("SELECT m FROM MyEntity m"); q.setHint(QueryHints.JDBC_FETCH_SIZE, "1000"); q.setHint(QueryHints.READ_ONLY, HintValues.TRUE); q.setHint("javax.persistence.cache.retrieveMode", "BYPASS"); log("Before GetResult - here everything is fine java allocates about 15 MB in memory"); List l = q.getResultList(); log("ArraySize: "+l.size() + " - 1,000,000 objects fetched from database take about 700 MB - 800 GB in memory"); List newList= new ArrayList(); for (MyEntity myEntity : l) { newList.add(myEntity); } log("List copied"); l.clear(); l = null; q = null; em.getTransaction().begin(); em.flush(); em.clear(); gc(5); log("Memory consumption will only descrease a little bit (~650 MB), because newList hold all entity object references with all jpa control overhead"); log("ArraySize: "+newList.size()); newList.clear(); newList = null; gc(5); log("Now all entity object references are cleared and so this raw payload and the jpa control payload is deallocated from memory ~ 20-50 MB remains in memory"); } private static void local() { log("Before creating"); List l = new ArrayList(); MyEntity m = new MyEntity(); for (int i = 1; i < 1000000; i++) { m = new MyEntity(); m.setBezeichnung(i+". Offset"); m.setDatum(new Date()); m.setEflow("N"); m.setEmail("sk@somewhere.de"); m.setFarbe("Green"); m.setFrist_id(i); m.setFrist_sperre_flag("Y"); m.setZeitbezug("Time ref."); l.add(m); } log("After creating, there are round about 150 MB memory consumption - the raw payload"); EntityManagerFactory factory = Persistence.createEntityManagerFactory("myPU"); EntityManager em = factory.createEntityManager(); for (Iterator iterator = l.iterator(); iterator.hasNext();) { em.persist(iterator.next()); } log("After adding to Persistence Context it explode to 400 MB"); gc(5); log("After garbage collection it is reduce to a normal and explainable 200 MB - 250 MB memory consumption ==> this is a acceptable memory consumption 100 MB raw data + 100 MB jpa control overhead"); log("ArraySize: "+l.size()); //have a reference on the list (otherwise the gc knows that there'll be no call on it an erase it from the memory too) } public static void main(String[] args) { em(); local(); } } ------------------------------------------------------------ The persistence.xml file ------------------------------------------------------------ org.eclipse.persistence.jpa.PersistenceProvider false helloworld.model.implementation.Employee test.MyEntity NONE ------------------------------------------------------------------------------------------ The run log: the values in [] are the real memory consumption with logging and profiling ------------------------------------------------------------------------------------------ [EL Finer]: 2011-06-21 10:27:34.406--Thread(Thread[main,5,main])--JavaSECMPInitializer - transformer is null. Tue Jun 21 10:27:35 CEST 2011 751 Before GetResult - here everything is fine java allocates about 15 MB in memory[15] Tue Jun 21 10:28:06 CEST 2011 452 ArraySize: 1000000 - 1,000,000 objects fetched from database take about 700 MB - 800 GB in memory[742] Tue Jun 21 10:28:07 CEST 2011 955 List copied[742] Tue Jun 21 10:28:08 CEST 2011 965 Wait 0 done[742] Tue Jun 21 10:28:10 CEST 2011 719 Wait 1 done[737] Tue Jun 21 10:28:12 CEST 2011 566 Wait 2 done[718] Tue Jun 21 10:28:14 CEST 2011 608 Wait 3 done[652] Tue Jun 21 10:28:16 CEST 2011 332 Wait 4 done[652] Tue Jun 21 10:28:17 CEST 2011 80 Memory consumption will only descrease a little bit (~650 MB), because newList hold all entity object references with all jpa control overhead[652] Tue Jun 21 10:28:17 CEST 2011 80 ArraySize: 1000000[652] Tue Jun 21 10:28:18 CEST 2011 85 Wait 0 done[652] Tue Jun 21 10:28:19 CEST 2011 272 Wait 1 done[84] Tue Jun 21 10:28:20 CEST 2011 312 Wait 2 done[50] Tue Jun 21 10:28:21 CEST 2011 348 Wait 3 done[50] Tue Jun 21 10:28:22 CEST 2011 388 Wait 4 done[24] Tue Jun 21 10:28:22 CEST 2011 424 Now all entity object references are cleared and so this raw payload and the jpa control payload is deallocated from memory ~ 20-50 MB remains in memory[24] Tue Jun 21 10:28:22 CEST 2011 424 Before creating[24] Tue Jun 21 10:28:24 CEST 2011 763 After creating, there are round about 150 MB memory consumption - the raw payload[173] [EL Finer]: 2011-06-21 10:28:24.764--Thread(Thread[main,5,main])--fixUNC: before fixing: url = file:/C:/Users/-------------/Desktop/jax/.workspace/HelloWorld/bin/, authority = , file = /C:/Users/-------------/Desktop/jax/.workspace/HelloWorld/bin/ (There is no English translation for this message.) [EL Finer]: 2011-06-21 10:28:24.764--Thread(Thread[main,5,main])--fixUNC: after fixing: url = file:/C:/Users/-------------/Desktop/jax/.workspace/HelloWorld/bin/, authority = null, file = /C:/Users/-------------/Desktop/jax/.workspace/HelloWorld/bin/ (There is no English translation for this message.) [EL Finer]: 2011-06-21 10:28:24.772--Thread(Thread[main,5,main])--JavaSECMPInitializer - predeploying myPU. [EL Finer]: 2011-06-21 10:28:24.809--Thread(Thread[main,5,main])--JavaSECMPInitializer - transformer is null. Tue Jun 21 10:31:17 CEST 2011 307 After adding to Persistence Context it explode to 400 MB[989] Tue Jun 21 10:31:18 CEST 2011 307 Wait 0 done[989] Tue Jun 21 10:31:20 CEST 2011 408 Wait 1 done[989] Tue Jun 21 10:31:22 CEST 2011 270 Wait 2 done[983] Tue Jun 21 10:31:23 CEST 2011 670 Wait 3 done[678] Tue Jun 21 10:31:25 CEST 2011 11 Wait 4 done[220] Tue Jun 21 10:31:25 CEST 2011 247 After garbage collection it is reduce to a normal and explainable 200 MB - 250 MB memory consumption ==> this is a acceptable memory consumption 100 MB raw data + 100 MB jpa control overhead[220] Tue Jun 21 10:31:25 CEST 2011 248 ArraySize: 999999[220] -------------------------------------------------------------------------------------------- The run log: the values in [] are the real memory consumption without logging and profiling -------------------------------------------------------------------------------------------- Tue Jun 21 10:45:19 CEST 2011 918 Before GetResult - here everything is fine java allocates about 15 MB in memory[15] Tue Jun 21 10:45:50 CEST 2011 594 ArraySize: 1000000 - 1,000,000 objects fetched from database take about 700 MB - 800 GB in memory[679] Tue Jun 21 10:45:52 CEST 2011 163 List copied[679] Tue Jun 21 10:45:53 CEST 2011 171 Wait 0 done[679] Tue Jun 21 10:45:55 CEST 2011 142 Wait 1 done[679] Tue Jun 21 10:45:57 CEST 2011 167 Wait 2 done[679] Tue Jun 21 10:45:58 CEST 2011 896 Wait 3 done[676] Tue Jun 21 10:46:00 CEST 2011 633 Wait 4 done[666] Tue Jun 21 10:46:01 CEST 2011 300 Memory consumption will only descrease a little bit (~650 MB), because newList hold all entity object references with all jpa control overhead[652] Tue Jun 21 10:46:01 CEST 2011 301 ArraySize: 1000000[652] Tue Jun 21 10:46:02 CEST 2011 305 Wait 0 done[652] Tue Jun 21 10:46:03 CEST 2011 483 Wait 1 done[49] Tue Jun 21 10:46:04 CEST 2011 520 Wait 2 done[49] Tue Jun 21 10:46:05 CEST 2011 558 Wait 3 done[23] Tue Jun 21 10:46:06 CEST 2011 592 Wait 4 done[23] Tue Jun 21 10:46:06 CEST 2011 638 Now all entity object references are cleared and so this raw payload and the jpa control payload is deallocated from memory ~ 20-50 MB remains in memory[23] Tue Jun 21 10:46:06 CEST 2011 638 Before creating[23] Tue Jun 21 10:46:08 CEST 2011 946 After creating, there are round about 150 MB memory consumption - the raw payload[171] Tue Jun 21 10:46:16 CEST 2011 434 After adding to Persistence Context it explode to 400 MB[417] Tue Jun 21 10:46:17 CEST 2011 435 Wait 0 done[417] Tue Jun 21 10:46:18 CEST 2011 750 Wait 1 done[417] Tue Jun 21 10:46:20 CEST 2011 35 Wait 2 done[398] Tue Jun 21 10:46:21 CEST 2011 325 Wait 3 done[327] Tue Jun 21 10:46:22 CEST 2011 620 Wait 4 done[220] Tue Jun 21 10:46:22 CEST 2011 857 After garbage collection it is reduce to a normal and explainable 200 MB - 250 MB memory consumption ==> this is a acceptable memory consumption 100 MB raw data + 100 MB jpa control overhead[220] Tue Jun 21 10:46:22 CEST 2011 858 ArraySize: 999999[220] ----------------------------------------------------------------- Statement/Question ----------------------------------------------------------------- - runtime args: -Xmx1024M -XX:MaxHeapFreeRatio=10 -XX:MinHeapFreeRatio=1 - Payload send from database: about 35 MB (because its an ANSI stream) - Raw entity object payload in memory should be ~100 MB (UTF-8 String, Date-Object, List References) - s - Why are the memory consumption such that high? - I tried * static weaving: no benefit for that use case * @ReadOnly-Annotation: no benefit for that use case