globalhs/apps/jretile.c

/* this module does the same thing as the SOI module retile, though not quite an exact port
 * translated by tim larson
 * future upgrade: keep track of missing inputs rather than initializing entire array to 0.
 */

#include <stdio.h>
#include <math.h>
#include <stdlib.h>
#include <time.h>
#include <sys/time.h>
#include <sys/resource.h>
#include "jsoc_main.h"
#include "drms_dsdsapi.h"
#include "errlog.h"

#define ARRLENGTH(ARR)  (sizeof(ARR)/sizeof(ARR[0]))

#define kNOTSPECIFIED "not specified"
#define QUAL_NODATA      (0x80000000)
#define QUAL_MIXEDCALVER (0x00000001)

#define MODES(L)        ((((L)+1)*(L))/2)
#define MINIMUM(X,Y)    (((X)<(Y))?(X):(Y))
#define MAXIMUM(X,Y)    (((X)>(Y))?(X):(Y))

char *module_name = "jretile";
char *cvsinfo_jretile = "cvsinfo: $Header: /home/cvsuser/cvsroot/JSOC/proj/globalhs/apps/jretile.c,v 1.11 2013/04/28 08:05:21 tplarson Exp $";

ModuleArgs_t module_args[] = 
{
   {ARG_STRING, "in",       NULL,          "input data records"},
   {ARG_STRING, "out",      NULL,          "output data series"},
   {ARG_STRING, "segin",    kNOTSPECIFIED, "input data segment"},
   {ARG_STRING, "segout",   kNOTSPECIFIED, "output data segment"},
   {ARG_STRING, "histlink", "HISTORY",     "name of link to ancillary dataseries for processing metadata"},
   {ARG_INT,    "MEMUSE",   "3",           "set to 0 to minimize use of file descriptors and memory, set to 1 to read slices and write whole files, 2 to read whole files and write slices, 3 to read whole files and write whole files."},  
   {ARG_INT,    "PERM",     "1",           "set to 0 for transient records, nonzero for permanent records"},
   {ARG_INT,    "VERB",     "1",           "option for level of verbosity: 0=only error and warning messages; >0=print messages outside of loop; >1=print messages inside loop; >2=debugging output", NULL},
   {ARG_INT,    "CALVERKEY","2",           "short integer bit mask determining which 4-bit fields of CALVER64 are permitted to change on input.  set the bit to disallow change of the corresponding nybble."},
   {ARG_INT,    "LMIN" ,    "0",           "minimum l in output", NULL},
   {ARG_INT,    "LMAX" ,    "0",           "maximum l in output", NULL},   /* if 0, default is LMAX of in_sds */
   {ARG_INT,    "LCHUNK",   "0",           "size of output chunk in l", NULL},/* if 0, is LMAX+1 */ 
   {ARG_TIME,   "TSTART",   NULL,          "start time of first output record"},
   {ARG_STRING, "TTOTAL",   NULL,          "total length of time in output"},
   {ARG_STRING, "TCHUNK",   kNOTSPECIFIED, "length of output timeseries"},
   {ARG_END}
};

#include "saveparm.c"
#include "timing.c"
#include "set_history.c"
#include "calversfunctions.c"

#include "jretile_manytofew.c"
#include "jretile_fewtomany.c"
#include "jretile_maxmem.c"

int DoIt(void)
{

  int status=0;
  char *inrecquery = NULL, *outseries = NULL;
  int lmin,lmax,lchunksize,lchunkfirst,lchunklast,nlchunks,ntimechunks,nrecsin,nrecsout;
  double nseconds,chunksecs;
  char *ttotal, *tchunk;
  DRMS_Record_t *tempoutrec = NULL;
//  DRMS_RecordSet_t *inrecset = NULL;
  int verbflag, memflag;

  inrecquery = (char *)cmdparams_get_str(&cmdparams, "in", &status);
  outseries = (char *)cmdparams_get_str(&cmdparams, "out", &status);
  lmin=cmdparams_get_int(&cmdparams, "LMIN", &status);
  lmax=cmdparams_get_int(&cmdparams, "LMAX", &status);
  lchunksize=cmdparams_get_int(&cmdparams, "LCHUNK", &status);
  if (lchunksize == 0)
    lchunksize=lmax+1;
  verbflag = cmdparams_get_int(&cmdparams, "VERB", &status);
  memflag = cmdparams_get_int(&cmdparams, "MEMUSE", &status);

  ttotal=(char *)cmdparams_get_str(&cmdparams, "TTOTAL", &status);
  status=drms_names_parseduration(&ttotal, &nseconds, 1);
  tchunk=(char *)cmdparams_get_str(&cmdparams, "TCHUNK", &status);
  if (strcmp(kNOTSPECIFIED, tchunk))
  {
    status=drms_names_parseduration(&tchunk, &chunksecs, 1);
  }
  else
    chunksecs=0;

  tempoutrec = drms_create_record(drms_env, outseries, DRMS_TRANSIENT, &status);
  if (status != DRMS_SUCCESS) 
  {
   fprintf(stderr,"ERROR: couldn't open a record in output dataseries %s, status = %d\n", outseries, status);
   return 1;
  }

  char *outchecklist[] = {"T_START", "QUALITY", "LMIN", "LMAX", "NDT"};
  DRMS_Keyword_t *outkeytest;
  int itest;
  for (itest=0; itest < ARRLENGTH(outchecklist); itest++)
  {
    outkeytest = hcon_lookup_lower(&tempoutrec->keywords, outchecklist[itest]);
    if (outkeytest == NULL || outkeytest->info->islink || outkeytest->info->recscope == 1)
    {
      fprintf(stderr, "ERROR: output keyword %s is either missing, constant, or a link\n", outchecklist[itest]);
      drms_close_record(tempoutrec, DRMS_FREE_RECORD);
      return 1;
    }
  }

  if (chunksecs == 0.0)
    chunksecs = drms_getkey_float(tempoutrec, "T_START_step", &status);
  drms_close_record(tempoutrec, DRMS_FREE_RECORD);

/*
  inrecset = drms_open_recordset(drms_env, inrecquery, &status);
  if (status != DRMS_SUCCESS || inrecset == NULL)
  {
    fprintf(stderr, "ERROR: problem opening input recordset: status = %d\n", status);
    return 1;
  }
  nrecsin = inrecset->n;
  drms_close_records(inrecset, DRMS_FREE_RECORD);
*/
  nrecsin = drms_count_records(drms_env, inrecquery, &status);
  if (status != DRMS_SUCCESS)
  {
    fprintf(stderr, "ERROR: problem counting input records: status = %d, nrecs = %d\n", status, nrecsin);
    return 1;
  }

  ntimechunks=nseconds/chunksecs;
  lchunkfirst = lmin/lchunksize;
  lchunklast = lmax/lchunksize;
  nlchunks = (lchunklast - lchunkfirst) + 1;
  nrecsout = nlchunks*ntimechunks;

  if (verbflag)
    printf("nrecsin = %d, nrecsout = %d, ", nrecsin, nrecsout);

  if (memflag==0)
  {
    if (nrecsin > nrecsout)
    {
      if (verbflag)
        printf("using jretile_manytofew()\n");
      status=jretile_manytofew();
    }
    else
    {
      if (verbflag)
        printf("using jretile_fewtomany()\n");
      status=jretile_fewtomany();
    }
  }
  else if (memflag==1)
  {
    if (verbflag)
      printf("forced use of jretile_fewtomany()\n");
    status=jretile_fewtomany();
  }
  else if (memflag==2)
  {
    if (verbflag)
      printf("forced use of jretile_manytofew()\n");
    status=jretile_manytofew();
  }
  else if (memflag==3)
  {
    if (verbflag)
      printf("using jretile_maxmem()\n");
    status=jretile_maxmem();
  }

  return status;

}
Revision:	1.12
Committed:	Fri Jun 6 13:04:19 2014 UTC (9 years, 3 months ago) by tplarson
Content type:	text/plain
Branch:	MAIN
CVS Tags:	globalhs_version_5, Ver_8-7, Ver_8-5, globalhs_version_23, globalhs_version_22, globalhs_version_21, globalhs_version_20, Ver_LATEST, globalhs_version_24, globalhs_version_8, globalhs_version_9, globalhs_version_4, Ver_9-41, globalhs_version_6, globalhs_version_7, Ver_9-5, Ver_8-8, globalhs_version_19, Ver_8-10, Ver_8-6, Ver_9-1, Ver_9-2, globalhs_version_12, globalhs_version_13, globalhs_version_10, globalhs_version_11, globalhs_version_16, globalhs_version_17, globalhs_version_14, globalhs_version_15, globalhs_version_18, Ver_9-4, Ver_9-3, Ver_8-11, Ver_8-12, Ver_9-0, HEAD
Changes since 1.11:	+31 -7 lines
Log Message:	add option to force use of particular retile routine, default to jretile_maxmem()
#	Content
1	/* this module does the same thing as the SOI module retile, though not quite an exact port
2	* translated by tim larson
3	* future upgrade: keep track of missing inputs rather than initializing entire array to 0.
4	*/
5
6	#include <stdio.h>
7	#include <math.h>
8	#include <stdlib.h>
9	#include <time.h>
10	#include <sys/time.h>
11	#include <sys/resource.h>
12	#include "jsoc_main.h"
13	#include "drms_dsdsapi.h"
14	#include "errlog.h"
15
16	#define ARRLENGTH(ARR) (sizeof(ARR)/sizeof(ARR[0]))
17
18	#define kNOTSPECIFIED "not specified"
19	#define QUAL_NODATA (0x80000000)
20	#define QUAL_MIXEDCALVER (0x00000001)
21
22	#define MODES(L) ((((L)+1)*(L))/2)
23	#define MINIMUM(X,Y) (((X)<(Y))?(X):(Y))
24	#define MAXIMUM(X,Y) (((X)>(Y))?(X):(Y))
25
26	char *module_name = "jretile";
27	char *cvsinfo_jretile = "cvsinfo: $Header: /home/cvsuser/cvsroot/JSOC/proj/globalhs/apps/jretile.c,v 1.11 2013/04/28 08:05:21 tplarson Exp $";
28
29	ModuleArgs_t module_args[] =
30	{
31	{ARG_STRING, "in", NULL, "input data records"},
32	{ARG_STRING, "out", NULL, "output data series"},
33	{ARG_STRING, "segin", kNOTSPECIFIED, "input data segment"},
34	{ARG_STRING, "segout", kNOTSPECIFIED, "output data segment"},
35	{ARG_STRING, "histlink", "HISTORY", "name of link to ancillary dataseries for processing metadata"},
36	{ARG_INT, "MEMUSE", "3", "set to 0 to minimize use of file descriptors and memory, set to 1 to read slices and write whole files, 2 to read whole files and write slices, 3 to read whole files and write whole files."},
37	{ARG_INT, "PERM", "1", "set to 0 for transient records, nonzero for permanent records"},
38	{ARG_INT, "VERB", "1", "option for level of verbosity: 0=only error and warning messages; >0=print messages outside of loop; >1=print messages inside loop; >2=debugging output", NULL},
39	{ARG_INT, "CALVERKEY","2", "short integer bit mask determining which 4-bit fields of CALVER64 are permitted to change on input. set the bit to disallow change of the corresponding nybble."},
40	{ARG_INT, "LMIN" , "0", "minimum l in output", NULL},
41	{ARG_INT, "LMAX" , "0", "maximum l in output", NULL}, /* if 0, default is LMAX of in_sds */
42	{ARG_INT, "LCHUNK", "0", "size of output chunk in l", NULL},/* if 0, is LMAX+1 */
43	{ARG_TIME, "TSTART", NULL, "start time of first output record"},
44	{ARG_STRING, "TTOTAL", NULL, "total length of time in output"},
45	{ARG_STRING, "TCHUNK", kNOTSPECIFIED, "length of output timeseries"},
46	{ARG_END}
47	};
48
49	#include "saveparm.c"
50	#include "timing.c"
51	#include "set_history.c"
52	#include "calversfunctions.c"
53
54	#include "jretile_manytofew.c"
55	#include "jretile_fewtomany.c"
56	#include "jretile_maxmem.c"
57
58	int DoIt(void)
59	{
60
61	int status=0;
62	char inrecquery = NULL, outseries = NULL;
63	int lmin,lmax,lchunksize,lchunkfirst,lchunklast,nlchunks,ntimechunks,nrecsin,nrecsout;
64	double nseconds,chunksecs;
65	char ttotal, tchunk;
66	DRMS_Record_t *tempoutrec = NULL;
67	// DRMS_RecordSet_t *inrecset = NULL;
68	int verbflag, memflag;
69
70	inrecquery = (char *)cmdparams_get_str(&cmdparams, "in", &status);
71	outseries = (char *)cmdparams_get_str(&cmdparams, "out", &status);
72	lmin=cmdparams_get_int(&cmdparams, "LMIN", &status);
73	lmax=cmdparams_get_int(&cmdparams, "LMAX", &status);
74	lchunksize=cmdparams_get_int(&cmdparams, "LCHUNK", &status);
75	if (lchunksize == 0)
76	lchunksize=lmax+1;
77	verbflag = cmdparams_get_int(&cmdparams, "VERB", &status);
78	memflag = cmdparams_get_int(&cmdparams, "MEMUSE", &status);
79
80	ttotal=(char *)cmdparams_get_str(&cmdparams, "TTOTAL", &status);
81	status=drms_names_parseduration(&ttotal, &nseconds, 1);
82	tchunk=(char *)cmdparams_get_str(&cmdparams, "TCHUNK", &status);
83	if (strcmp(kNOTSPECIFIED, tchunk))
84	{
85	status=drms_names_parseduration(&tchunk, &chunksecs, 1);
86	}
87	else
88	chunksecs=0;
89
90	tempoutrec = drms_create_record(drms_env, outseries, DRMS_TRANSIENT, &status);
91	if (status != DRMS_SUCCESS)
92	{
93	fprintf(stderr,"ERROR: couldn't open a record in output dataseries %s, status = %d\n", outseries, status);
94	return 1;
95	}
96
97	char *outchecklist[] = {"T_START", "QUALITY", "LMIN", "LMAX", "NDT"};
98	DRMS_Keyword_t *outkeytest;
99	int itest;
100	for (itest=0; itest < ARRLENGTH(outchecklist); itest++)
101	{
102	outkeytest = hcon_lookup_lower(&tempoutrec->keywords, outchecklist[itest]);
103	if (outkeytest == NULL \|\| outkeytest->info->islink \|\| outkeytest->info->recscope == 1)
104	{
105	fprintf(stderr, "ERROR: output keyword %s is either missing, constant, or a link\n", outchecklist[itest]);
106	drms_close_record(tempoutrec, DRMS_FREE_RECORD);
107	return 1;
108	}
109	}
110
111	if (chunksecs == 0.0)
112	chunksecs = drms_getkey_float(tempoutrec, "T_START_step", &status);
113	drms_close_record(tempoutrec, DRMS_FREE_RECORD);
114
115	/*
116	inrecset = drms_open_recordset(drms_env, inrecquery, &status);
117	if (status != DRMS_SUCCESS \|\| inrecset == NULL)
118	{
119	fprintf(stderr, "ERROR: problem opening input recordset: status = %d\n", status);
120	return 1;
121	}
122	nrecsin = inrecset->n;
123	drms_close_records(inrecset, DRMS_FREE_RECORD);
124	*/
125	nrecsin = drms_count_records(drms_env, inrecquery, &status);
126	if (status != DRMS_SUCCESS)
127	{
128	fprintf(stderr, "ERROR: problem counting input records: status = %d, nrecs = %d\n", status, nrecsin);
129	return 1;
130	}
131
132	ntimechunks=nseconds/chunksecs;
133	lchunkfirst = lmin/lchunksize;
134	lchunklast = lmax/lchunksize;
135	nlchunks = (lchunklast - lchunkfirst) + 1;
136	nrecsout = nlchunks*ntimechunks;
137
138	if (verbflag)
139	printf("nrecsin = %d, nrecsout = %d, ", nrecsin, nrecsout);
140
141	if (memflag==0)
142	{
143	if (nrecsin > nrecsout)
144	{
145	if (verbflag)
146	printf("using jretile_manytofew()\n");
147	status=jretile_manytofew();
148	}
149	else
150	{
151	if (verbflag)
152	printf("using jretile_fewtomany()\n");
153	status=jretile_fewtomany();
154	}
155	}
156	else if (memflag==1)
157	{
158	if (verbflag)
159	printf("forced use of jretile_fewtomany()\n");
160	status=jretile_fewtomany();
161	}
162	else if (memflag==2)
163	{
164	if (verbflag)
165	printf("forced use of jretile_manytofew()\n");
166	status=jretile_manytofew();
167	}
168	else if (memflag==3)
169	{
170	if (verbflag)
171	printf("using jretile_maxmem()\n");
172	status=jretile_maxmem();
173	}
174
175	return status;
176
177	}