Actual source code: jostle.c

  1: #define PETSCMAT_DLL

 3:  #include src/mat/impls/adj/mpi/mpiadj.h

  5: #ifdef PETSC_HAVE_UNISTD_H
  6: #include <unistd.h>
  7: #endif

  9: #ifdef PETSC_HAVE_STDLIB_H
 10: #include <stdlib.h>
 11: #endif

 13: #include "petscfix.h"


 17: #include "jostle.h"
 18: /* this function is not declared in 'jostle.h' */


 23: typedef struct {
 24:     int output;
 25:     int coarse_seq;
 26:     int nbvtxcoarsed;           /* number of vertices for the coarse graph */
 27:     char *mesg_log;
 28: } MatPartitioning_Jostle;

 30: #define SIZE_LOG 10000          /* size of buffer for msg_log */

 34: static PetscErrorCode MatPartitioningApply_Jostle(MatPartitioning part, IS * partitioning)
 35: {
 37:     int  size, rank, i;
 38:     Mat mat = part->adj, matMPI;
 39:     Mat_MPIAdj *adj = (Mat_MPIAdj *) mat->data;
 40:     MatPartitioning_Jostle *jostle_struct =
 41:         (MatPartitioning_Jostle *) part->data;
 42:     PetscTruth flg;
 43: #ifdef PETSC_HAVE_UNISTD_H
 44:     int fd_stdout, fd_pipe[2], count;
 45: #endif


 49:     /* check that the number of partitions is equal to the number of processors */
 50:     MPI_Comm_rank(mat->comm, &rank);
 51:     MPI_Comm_size(mat->comm, &size);
 52:     if (part->n != size) {
 53:         SETERRQ(PETSC_ERR_SUP, "Supports exactly one domain per processor");
 54:     }

 56:     /* convert adjacency matrix to MPIAdj if needed*/
 57:     PetscTypeCompare((PetscObject) mat, MATMPIADJ, &flg);
 58:     if (!flg) {
 59:         MatConvert(mat, MATMPIADJ, MAT_INITIAL_MATRIX, &matMPI);
 60:     } else
 61:         matMPI = mat;

 63:     adj = (Mat_MPIAdj *) matMPI->data;  /* adj contains adjacency graph */
 64:     {
 65:         /* definition of Jostle library arguments */
 66:         int nnodes = matMPI->M; /* number of vertices in full graph */
 67:         int offset = 0;         /* 0 for C array indexing */
 68:         int core = matMPI->m;
 69:         int halo = 0;           /* obsolete with contiguous format */
 70:         int *index_jostle;      /* contribution of each processor */
 71:         int nparts = part->n;
 72:         int *part_wt = NULL;

 74:         int *partition;         /* set number of each vtx (length n) */
 75:         int *degree;            /* degree for each core nodes */
 76:         int *edges = adj->j;
 77:         int *node_wt = NULL;    /* nodes weights */
 78:         int *edge_wt = NULL;    /* edges weights */
 79:         double *coords = NULL;  /* not used (cf jostle documentation) */

 81:         int local_nedges = adj->nz;
 82:         int dimension = 0;      /* not used */
 83:         int output_level = jostle_struct->output;
 84:         char env_str[256];

 86:         /* allocate index_jostle */
 87:         PetscMalloc(nparts * sizeof(int), &index_jostle);

 89:         /* compute number of core nodes for each one */
 90:         for (i = 0; i < nparts - 1; i++)
 91:             index_jostle[i] = adj->rowners[i + 1] - adj->rowners[i];
 92:         index_jostle[nparts - 1] = nnodes - adj->rowners[nparts - 1];

 94:         /* allocate the partition vector */
 95:         PetscMalloc(core * sizeof(int), &partition);

 97:         /* build the degree vector and the local_nedges value */
 98:         PetscMalloc(core * sizeof(int), &degree);
 99:         for (i = 0; i < core; i++)
100:             degree[i] = adj->i[i + 1] - adj->i[i];

102:         /* library call */
103:         pjostle_init(&size, &rank);
104:         pjostle_comm(&matMPI->comm);
105:         jostle_env("format = contiguous");
106:         jostle_env("timer = off");

108:         sprintf(env_str, "threshold = %d", jostle_struct->nbvtxcoarsed);
109:         jostle_env(env_str);

111:         if (jostle_struct->coarse_seq)
112:           jostle_env("matching = local");

114:         /* redirect output */
115: #ifdef PETSC_HAVE_UNISTD_H
116:         fd_stdout = dup(1);
117:         pipe(fd_pipe);
118:         close(1);
119:         dup2(fd_pipe[1], 1);
120: #endif

122:         pjostle(&nnodes, &offset, &core, &halo, index_jostle, degree, node_wt,
123:             partition, &local_nedges, edges, edge_wt, &nparts,
124:             part_wt, &output_level, &dimension, coords);

126:         printf("Jostle Partitioner statistics\ncut : %d, balance : %f, runtime : %f, mem used : %d\n",
127:             jostle_cut(), jostle_bal(), jostle_tim(), jostle_mem());

129: #ifdef PETSC_HAVE_UNISTD_H
130:         PetscMalloc(SIZE_LOG * sizeof(char), &(jostle_struct->mesg_log));
131:         fflush(stdout);
132:         count = read(fd_pipe[0], jostle_struct->mesg_log, (SIZE_LOG - 1) * sizeof(char));
133:         if (count < 0)
134:             count = 0;
135:         jostle_struct->mesg_log[count] = 0;
136:         close(1);
137:         dup2(fd_stdout, 1);
138:         close(fd_stdout);
139:         close(fd_pipe[0]);
140:         close(fd_pipe[1]);
141: #endif

143:         /* We free the memory used by jostle */
144:         PetscFree(index_jostle);
145:         PetscFree(degree);

147:         /* Creation of the index set */
148:         ISCreateGeneral(part->comm, mat->m, partition, partitioning);

150:         if (matMPI != mat) {
151:             MatDestroy(matMPI);
152:         }

154:         PetscFree(partition);
155:     }

157:     return(0);
158: }


163: PetscErrorCode MatPartitioningView_Jostle(MatPartitioning part, PetscViewer viewer)
164: {
165:   MatPartitioning_Jostle *jostle_struct = (MatPartitioning_Jostle *) part->data;
166:   PetscErrorCode         ierr;
167:   PetscTruth             iascii;

170:   PetscTypeCompare((PetscObject) viewer, PETSC_VIEWER_ASCII, &iascii);
171:   if (iascii) {
172:     if (jostle_struct->mesg_log) {
173:       PetscViewerASCIIPrintf(viewer, "%s\n", jostle_struct->mesg_log);
174:     }
175:   } else {
176:     SETERRQ1(PETSC_ERR_SUP, "Viewer type %s not supported for this Jostle partitioner",((PetscObject) viewer)->type_name);
177:   }
178:   return(0);
179: }

183: /*@
184:     MatPartitioningJostleSetCoarseLevel - Set the coarse level 
185:     
186:   Input Parameter:
187: .  part - the partitioning context
188: .  level - the coarse level in range [0.0,1.0]

190:    Level: advanced

192: @*/
193: PetscErrorCode  MatPartitioningJostleSetCoarseLevel(MatPartitioning part, PetscReal level)
194: {
195:     MatPartitioning_Jostle *jostle_struct = (MatPartitioning_Jostle *) part->data;


199:     if (level < 0.0 || level > 1.0) {
200:         SETERRQ(PETSC_ERR_ARG_OUTOFRANGE,
201:             "Jostle: level of coarsening out of range [0.0-1.0]");
202:     } else
203:         jostle_struct->nbvtxcoarsed = (int)(part->adj->N * level);

205:     if (jostle_struct->nbvtxcoarsed < 20)
206:         jostle_struct->nbvtxcoarsed = 20;

208:     return(0);
209: }

213: /*@
214:      MatPartitioningJostleSetCoarseSequential - Use the sequential code to 
215:          do the partitioning of the coarse grid.

217:   Input Parameter:
218: .  part - the partitioning context

220:    Level: advanced

222: @*/
223: PetscErrorCode  MatPartitioningJostleSetCoarseSequential(MatPartitioning part)
224: {
225:     MatPartitioning_Jostle *jostle_struct =
226:         (MatPartitioning_Jostle *) part->data;
228:     jostle_struct->coarse_seq = 1;
229:     return(0);
230: }

234: PetscErrorCode MatPartitioningSetFromOptions_Jostle(MatPartitioning part)
235: {
237:     PetscTruth flag;
238:     PetscReal level;

241:     PetscOptionsHead("Set Jostle partitioning options");

243:     PetscOptionsReal("-mat_partitioning_jostle_coarse_level",
244:         "Coarse level", "MatPartitioningJostleSetCoarseLevel", 0,
245:         &level, &flag);
246:     if (flag)
247:         MatPartitioningJostleSetCoarseLevel(part, level);

249:     PetscOptionsName("-mat_partitioning_jostle_coarse_sequential",
250:         "Use sequential coarse partitioner",
251:         "MatPartitioningJostleSetCoarseSequential", &flag);
252:     if (flag) {
253:         MatPartitioningJostleSetCoarseSequential(part);
254:     }

256:     PetscOptionsTail();
257:     return(0);
258: }


263: PetscErrorCode MatPartitioningDestroy_Jostle(MatPartitioning part)
264: {
265:     MatPartitioning_Jostle *jostle_struct = (MatPartitioning_Jostle *) part->data;
266:     PetscErrorCode         ierr;

269:     PetscFree(jostle_struct->mesg_log);
270:     PetscFree(jostle_struct);
271:     return(0);
272: }

277: /*@C
278:    MAT_PARTITIONING_JOSTLE - Creates a partitioning context via the external package Jostle.

280:    Collective on MPI_Comm

282:    Input Parameter:
283: .  part - the partitioning context

285:    Options Database Keys:
286: +  -mat_partitioning_jostle_coarse_level <0>: Coarse level (MatPartitioningJostleSetCoarseLevel)
287: -  -mat_partitioning_jostle_coarse_sequential: Use sequential coarse partitioner (MatPartitioningJostleSetCoarseSequential)

289:    Level: beginner

291:    Notes: See http://www.gre.ac.uk/~c.walshaw/jostle/

293: .keywords: Partitioning, create, context

295: .seealso: MatPartitioningSetType(), MatPartitioningType

297: @*/
298: PetscErrorCode  MatPartitioningCreate_Jostle(MatPartitioning part)
299: {
301:     MatPartitioning_Jostle *jostle_struct;

304:     PetscNew(MatPartitioning_Jostle, &jostle_struct);

306:     jostle_struct->nbvtxcoarsed = 20;
307:     jostle_struct->output = 0;
308:     jostle_struct->coarse_seq = 0;
309:     jostle_struct->mesg_log = NULL;

311:     part->ops->apply = MatPartitioningApply_Jostle;
312:     part->ops->view = MatPartitioningView_Jostle;
313:     part->ops->destroy = MatPartitioningDestroy_Jostle;
314:     part->ops->setfromoptions = MatPartitioningSetFromOptions_Jostle;
315:     part->data = (void*) jostle_struct;

317:     return(0);
318: }