globalhs/apps/jretile.c

/* this module does the same thing as the SOI module retile, though not quite an exact port
 * translated by tim larson
 * future upgrade: keep track of missing inputs rather than initializing entire array to 0.
 */

#include <stdio.h>
#include <math.h>
#include <stdlib.h>
#include <time.h>
#include <sys/time.h>
#include <sys/resource.h>
#include "jsoc_main.h"
#include "drms_dsdsapi.h"
#include "errlog.h"

#define ARRLENGTH(ARR)  (sizeof(ARR)/sizeof(ARR[0]))

#define kNOTSPECIFIED "not specified"
#define QUAL_NODATA      (0x80000000)
#define QUAL_MIXEDCALVER (0x00000001)

#define MODES(L)        ((((L)+1)*(L))/2)
#define MINIMUM(X,Y)    (((X)<(Y))?(X):(Y))
#define MAXIMUM(X,Y)    (((X)>(Y))?(X):(Y))

char *module_name = "jretile";
char *cvsinfo_jretile = "cvsinfo: $Header: /home/cvsuser/cvsroot/JSOC/proj/globalhs/apps/jretile.c,v 1.11 2013/04/28 08:05:21 tplarson Exp $";

ModuleArgs_t module_args[] = 
{
   {ARG_STRING, "in",       NULL,          "input data records"},
   {ARG_STRING, "out",      NULL,          "output data series"},
   {ARG_STRING, "segin",    kNOTSPECIFIED, "input data segment"},
   {ARG_STRING, "segout",   kNOTSPECIFIED, "output data segment"},
   {ARG_STRING, "histlink", "HISTORY",     "name of link to ancillary dataseries for processing metadata"},
   {ARG_INT,    "MEMUSE",   "3",           "set to 0 to minimize use of file descriptors and memory, set to 1 to read slices and write whole files, 2 to read whole files and write slices, 3 to read whole files and write whole files."},  
   {ARG_INT,    "PERM",     "1",           "set to 0 for transient records, nonzero for permanent records"},
   {ARG_INT,    "VERB",     "1",           "option for level of verbosity: 0=only error and warning messages; >0=print messages outside of loop; >1=print messages inside loop; >2=debugging output", NULL},
   {ARG_INT,    "CALVERKEY","2",           "short integer bit mask determining which 4-bit fields of CALVER64 are permitted to change on input.  set the bit to disallow change of the corresponding nybble."},
   {ARG_INT,    "LMIN" ,    "0",           "minimum l in output", NULL},
   {ARG_INT,    "LMAX" ,    "0",           "maximum l in output", NULL},   /* if 0, default is LMAX of in_sds */
   {ARG_INT,    "LCHUNK",   "0",           "size of output chunk in l", NULL},/* if 0, is LMAX+1 */ 
   {ARG_TIME,   "TSTART",   NULL,          "start time of first output record"},
   {ARG_STRING, "TTOTAL",   NULL,          "total length of time in output"},
   {ARG_STRING, "TCHUNK",   kNOTSPECIFIED, "length of output timeseries"},
   {ARG_END}
};

#include "saveparm.c"
#include "timing.c"
#include "set_history.c"
#include "calversfunctions.c"

#include "jretile_manytofew.c"
#include "jretile_fewtomany.c"
#include "jretile_maxmem.c"

int DoIt(void)
{

  int status=0;
  char *inrecquery = NULL, *outseries = NULL;
  int lmin,lmax,lchunksize,lchunkfirst,lchunklast,nlchunks,ntimechunks,nrecsin,nrecsout;
  double nseconds,chunksecs;
  char *ttotal, *tchunk;
  DRMS_Record_t *tempoutrec = NULL;
//  DRMS_RecordSet_t *inrecset = NULL;
  int verbflag, memflag;

  inrecquery = (char *)cmdparams_get_str(&cmdparams, "in", &status);
  outseries = (char *)cmdparams_get_str(&cmdparams, "out", &status);
  lmin=cmdparams_get_int(&cmdparams, "LMIN", &status);
  lmax=cmdparams_get_int(&cmdparams, "LMAX", &status);
  lchunksize=cmdparams_get_int(&cmdparams, "LCHUNK", &status);
  if (lchunksize == 0)
    lchunksize=lmax+1;
  verbflag = cmdparams_get_int(&cmdparams, "VERB", &status);
  memflag = cmdparams_get_int(&cmdparams, "MEMUSE", &status);

  ttotal=(char *)cmdparams_get_str(&cmdparams, "TTOTAL", &status);
  status=drms_names_parseduration(&ttotal, &nseconds, 1);
  tchunk=(char *)cmdparams_get_str(&cmdparams, "TCHUNK", &status);
  if (strcmp(kNOTSPECIFIED, tchunk))
  {
    status=drms_names_parseduration(&tchunk, &chunksecs, 1);
  }
  else
    chunksecs=0;

  tempoutrec = drms_create_record(drms_env, outseries, DRMS_TRANSIENT, &status);
  if (status != DRMS_SUCCESS) 
  {
   fprintf(stderr,"ERROR: couldn't open a record in output dataseries %s, status = %d\n", outseries, status);
   return 1;
  }

  char *outchecklist[] = {"T_START", "QUALITY", "LMIN", "LMAX", "NDT"};
  DRMS_Keyword_t *outkeytest;
  int itest;
  for (itest=0; itest < ARRLENGTH(outchecklist); itest++)
  {
    outkeytest = hcon_lookup_lower(&tempoutrec->keywords, outchecklist[itest]);
    if (outkeytest == NULL || outkeytest->info->islink || outkeytest->info->recscope == 1)
    {
      fprintf(stderr, "ERROR: output keyword %s is either missing, constant, or a link\n", outchecklist[itest]);
      drms_close_record(tempoutrec, DRMS_FREE_RECORD);
      return 1;
    }
  }

  if (chunksecs == 0.0)
    chunksecs = drms_getkey_float(tempoutrec, "T_START_step", &status);
  drms_close_record(tempoutrec, DRMS_FREE_RECORD);

/*
  inrecset = drms_open_recordset(drms_env, inrecquery, &status);
  if (status != DRMS_SUCCESS || inrecset == NULL)
  {
    fprintf(stderr, "ERROR: problem opening input recordset: status = %d\n", status);
    return 1;
  }
  nrecsin = inrecset->n;
  drms_close_records(inrecset, DRMS_FREE_RECORD);
*/
  nrecsin = drms_count_records(drms_env, inrecquery, &status);
  if (status != DRMS_SUCCESS)
  {
    fprintf(stderr, "ERROR: problem counting input records: status = %d, nrecs = %d\n", status, nrecsin);
    return 1;
  }

  ntimechunks=nseconds/chunksecs;
  lchunkfirst = lmin/lchunksize;
  lchunklast = lmax/lchunksize;
  nlchunks = (lchunklast - lchunkfirst) + 1;
  nrecsout = nlchunks*ntimechunks;

  if (verbflag)
    printf("nrecsin = %d, nrecsout = %d, ", nrecsin, nrecsout);

  if (memflag==0)
  {
    if (nrecsin > nrecsout)
    {
      if (verbflag)
        printf("using jretile_manytofew()\n");
      status=jretile_manytofew();
    }
    else
    {
      if (verbflag)
        printf("using jretile_fewtomany()\n");
      status=jretile_fewtomany();
    }
  }
  else if (memflag==1)
  {
    if (verbflag)
      printf("forced use of jretile_fewtomany()\n");
    status=jretile_fewtomany();
  }
  else if (memflag==2)
  {
    if (verbflag)
      printf("forced use of jretile_manytofew()\n");
    status=jretile_manytofew();
  }
  else if (memflag==3)
  {
    if (verbflag)
      printf("using jretile_maxmem()\n");
    status=jretile_maxmem();
  }

  return status;

}
Revision:	1.12
Committed:	Fri Jun 6 13:04:19 2014 UTC (9 years, 3 months ago) by tplarson
Content type:	text/plain
Branch:	MAIN
CVS Tags:	globalhs_version_5, Ver_8-7, Ver_8-5, globalhs_version_23, globalhs_version_22, globalhs_version_21, globalhs_version_20, Ver_LATEST, globalhs_version_24, globalhs_version_8, globalhs_version_9, globalhs_version_4, Ver_9-41, globalhs_version_6, globalhs_version_7, Ver_9-5, Ver_8-8, globalhs_version_19, Ver_8-10, Ver_8-6, Ver_9-1, Ver_9-2, globalhs_version_12, globalhs_version_13, globalhs_version_10, globalhs_version_11, globalhs_version_16, globalhs_version_17, globalhs_version_14, globalhs_version_15, globalhs_version_18, Ver_9-4, Ver_9-3, Ver_8-11, Ver_8-12, Ver_9-0, HEAD
Changes since 1.11:	+31 -7 lines
Log Message:	add option to force use of particular retile routine, default to jretile_maxmem()
#	User	Rev	Content
1	tplarson	1.1	/* this module does the same thing as the SOI module retile, though not quite an exact port
2	tplarson	1.10	* translated by tim larson
3	tplarson	1.1	* future upgrade: keep track of missing inputs rather than initializing entire array to 0.
4			*/
5
6			#include <stdio.h>
7			#include <math.h>
8			#include <stdlib.h>
9			#include <time.h>
10			#include <sys/time.h>
11			#include <sys/resource.h>
12			#include "jsoc_main.h"
13			#include "drms_dsdsapi.h"
14			#include "errlog.h"
15
16			#define ARRLENGTH(ARR) (sizeof(ARR)/sizeof(ARR[0]))
17
18			#define kNOTSPECIFIED "not specified"
19	tplarson	1.10	#define QUAL_NODATA (0x80000000)
20			#define QUAL_MIXEDCALVER (0x00000001)
21
22			#define MODES(L) ((((L)+1)*(L))/2)
23			#define MINIMUM(X,Y) (((X)<(Y))?(X):(Y))
24			#define MAXIMUM(X,Y) (((X)>(Y))?(X):(Y))
25	tplarson	1.1
26			char *module_name = "jretile";
27	tplarson	1.12	char *cvsinfo_jretile = "cvsinfo: $Header: /home/cvsuser/cvsroot/JSOC/proj/globalhs/apps/jretile.c,v 1.11 2013/04/28 08:05:21 tplarson Exp $";
28	tplarson	1.1
29			ModuleArgs_t module_args[] =
30			{
31			{ARG_STRING, "in", NULL, "input data records"},
32			{ARG_STRING, "out", NULL, "output data series"},
33			{ARG_STRING, "segin", kNOTSPECIFIED, "input data segment"},
34			{ARG_STRING, "segout", kNOTSPECIFIED, "output data segment"},
35			{ARG_STRING, "histlink", "HISTORY", "name of link to ancillary dataseries for processing metadata"},
36	tplarson	1.12	{ARG_INT, "MEMUSE", "3", "set to 0 to minimize use of file descriptors and memory, set to 1 to read slices and write whole files, 2 to read whole files and write slices, 3 to read whole files and write whole files."},
37	tplarson	1.1	{ARG_INT, "PERM", "1", "set to 0 for transient records, nonzero for permanent records"},
38			{ARG_INT, "VERB", "1", "option for level of verbosity: 0=only error and warning messages; >0=print messages outside of loop; >1=print messages inside loop; >2=debugging output", NULL},
39	tplarson	1.10	{ARG_INT, "CALVERKEY","2", "short integer bit mask determining which 4-bit fields of CALVER64 are permitted to change on input. set the bit to disallow change of the corresponding nybble."},
40	tplarson	1.1	{ARG_INT, "LMIN" , "0", "minimum l in output", NULL},
41			{ARG_INT, "LMAX" , "0", "maximum l in output", NULL}, /* if 0, default is LMAX of in_sds */
42			{ARG_INT, "LCHUNK", "0", "size of output chunk in l", NULL},/* if 0, is LMAX+1 */
43			{ARG_TIME, "TSTART", NULL, "start time of first output record"},
44			{ARG_STRING, "TTOTAL", NULL, "total length of time in output"},
45			{ARG_STRING, "TCHUNK", kNOTSPECIFIED, "length of output timeseries"},
46			{ARG_END}
47			};
48
49			#include "saveparm.c"
50			#include "timing.c"
51	tplarson	1.6	#include "set_history.c"
52	tplarson	1.10	#include "calversfunctions.c"
53	tplarson	1.1
54	tplarson	1.8	#include "jretile_manytofew.c"
55			#include "jretile_fewtomany.c"
56	tplarson	1.12	#include "jretile_maxmem.c"
57	tplarson	1.1
58			int DoIt(void)
59			{
60
61	tplarson	1.8	int status=0;
62			char inrecquery = NULL, outseries = NULL;
63			int lmin,lmax,lchunksize,lchunkfirst,lchunklast,nlchunks,ntimechunks,nrecsin,nrecsout;
64			double nseconds,chunksecs;
65			char ttotal, tchunk;
66			DRMS_Record_t *tempoutrec = NULL;
67	tplarson	1.10	// DRMS_RecordSet_t *inrecset = NULL;
68	tplarson	1.12	int verbflag, memflag;
69	tplarson	1.1
70	tplarson	1.8	inrecquery = (char *)cmdparams_get_str(&cmdparams, "in", &status);
71			outseries = (char *)cmdparams_get_str(&cmdparams, "out", &status);
72			lmin=cmdparams_get_int(&cmdparams, "LMIN", &status);
73			lmax=cmdparams_get_int(&cmdparams, "LMAX", &status);
74			lchunksize=cmdparams_get_int(&cmdparams, "LCHUNK", &status);
75			if (lchunksize == 0)
76			lchunksize=lmax+1;
77			verbflag = cmdparams_get_int(&cmdparams, "VERB", &status);
78	tplarson	1.12	memflag = cmdparams_get_int(&cmdparams, "MEMUSE", &status);
79	tplarson	1.1
80	tplarson	1.8	ttotal=(char *)cmdparams_get_str(&cmdparams, "TTOTAL", &status);
81	arta	1.5	status=drms_names_parseduration(&ttotal, &nseconds, 1);
82	tplarson	1.8	tchunk=(char *)cmdparams_get_str(&cmdparams, "TCHUNK", &status);
83	tplarson	1.1	if (strcmp(kNOTSPECIFIED, tchunk))
84			{
85	arta	1.5	status=drms_names_parseduration(&tchunk, &chunksecs, 1);
86	tplarson	1.1	}
87			else
88			chunksecs=0;
89
90	tplarson	1.8	tempoutrec = drms_create_record(drms_env, outseries, DRMS_TRANSIENT, &status);
91	tplarson	1.1	if (status != DRMS_SUCCESS)
92			{
93			fprintf(stderr,"ERROR: couldn't open a record in output dataseries %s, status = %d\n", outseries, status);
94			return 1;
95			}
96
97	tplarson	1.8	char *outchecklist[] = {"T_START", "QUALITY", "LMIN", "LMAX", "NDT"};
98			DRMS_Keyword_t *outkeytest;
99	tplarson	1.1	int itest;
100			for (itest=0; itest < ARRLENGTH(outchecklist); itest++)
101			{
102	tplarson	1.8	outkeytest = hcon_lookup_lower(&tempoutrec->keywords, outchecklist[itest]);
103	tplarson	1.6	if (outkeytest == NULL \|\| outkeytest->info->islink \|\| outkeytest->info->recscope == 1)
104	tplarson	1.1	{
105			fprintf(stderr, "ERROR: output keyword %s is either missing, constant, or a link\n", outchecklist[itest]);
106	tplarson	1.3	drms_close_record(tempoutrec, DRMS_FREE_RECORD);
107	tplarson	1.6	return 1;
108	tplarson	1.1	}
109			}
110
111	tplarson	1.3	if (chunksecs == 0.0)
112	tplarson	1.8	chunksecs = drms_getkey_float(tempoutrec, "T_START_step", &status);
113	tplarson	1.3	drms_close_record(tempoutrec, DRMS_FREE_RECORD);
114
115	tplarson	1.10	/*
116	tplarson	1.8	inrecset = drms_open_recordset(drms_env, inrecquery, &status);
117	tplarson	1.6	if (status != DRMS_SUCCESS \|\| inrecset == NULL)
118	tplarson	1.1	{
119			fprintf(stderr, "ERROR: problem opening input recordset: status = %d\n", status);
120	tplarson	1.6	return 1;
121	tplarson	1.1	}
122	tplarson	1.8	nrecsin = inrecset->n;
123			drms_close_records(inrecset, DRMS_FREE_RECORD);
124	tplarson	1.10	*/
125			nrecsin = drms_count_records(drms_env, inrecquery, &status);
126			if (status != DRMS_SUCCESS)
127			{
128			fprintf(stderr, "ERROR: problem counting input records: status = %d, nrecs = %d\n", status, nrecsin);
129			return 1;
130			}
131	tplarson	1.7
132	tplarson	1.8	ntimechunks=nseconds/chunksecs;
133	tplarson	1.7	lchunkfirst = lmin/lchunksize;
134			lchunklast = lmax/lchunksize;
135	tplarson	1.8	nlchunks = (lchunklast - lchunkfirst) + 1;
136			nrecsout = nlchunks*ntimechunks;
137	tplarson	1.7
138	tplarson	1.8	if (verbflag)
139			printf("nrecsin = %d, nrecsout = %d, ", nrecsin, nrecsout);
140	tplarson	1.7
141	tplarson	1.12	if (memflag==0)
142			{
143			if (nrecsin > nrecsout)
144			{
145			if (verbflag)
146			printf("using jretile_manytofew()\n");
147			status=jretile_manytofew();
148			}
149			else
150			{
151			if (verbflag)
152			printf("using jretile_fewtomany()\n");
153			status=jretile_fewtomany();
154			}
155			}
156			else if (memflag==1)
157	tplarson	1.1	{
158	tplarson	1.8	if (verbflag)
159	tplarson	1.12	printf("forced use of jretile_fewtomany()\n");
160			status=jretile_fewtomany();
161			}
162			else if (memflag==2)
163			{
164			if (verbflag)
165			printf("forced use of jretile_manytofew()\n");
166	tplarson	1.8	status=jretile_manytofew();
167	tplarson	1.7	}
168	tplarson	1.12	else if (memflag==3)
169	tplarson	1.1	{
170	tplarson	1.8	if (verbflag)
171	tplarson	1.12	printf("using jretile_maxmem()\n");
172			status=jretile_maxmem();
173	tplarson	1.1	}
174
175	tplarson	1.8	return status;
176	tplarson	1.1
177			}