Actual source code: jostle.c
1: #define PETSCMAT_DLL
3: #include src/mat/impls/adj/mpi/mpiadj.h
5: #ifdef PETSC_HAVE_UNISTD_H
6: #include <unistd.h>
7: #endif
9: #ifdef PETSC_HAVE_STDLIB_H
10: #include <stdlib.h>
11: #endif
13: #include "petscfix.h"
17: #include "jostle.h"
18: /* this function is not declared in 'jostle.h' */
23: typedef struct {
24: int output;
25: int coarse_seq;
26: int nbvtxcoarsed; /* number of vertices for the coarse graph */
27: char *mesg_log;
28: } MatPartitioning_Jostle;
30: #define SIZE_LOG 10000 /* size of buffer for msg_log */
34: static PetscErrorCode MatPartitioningApply_Jostle(MatPartitioning part, IS * partitioning)
35: {
37: int size, rank, i;
38: Mat mat = part->adj, matMPI;
39: Mat_MPIAdj *adj = (Mat_MPIAdj *) mat->data;
40: MatPartitioning_Jostle *jostle_struct =
41: (MatPartitioning_Jostle *) part->data;
42: PetscTruth flg;
43: #ifdef PETSC_HAVE_UNISTD_H
44: int fd_stdout, fd_pipe[2], count;
45: #endif
49: /* check that the number of partitions is equal to the number of processors */
50: MPI_Comm_rank(mat->comm, &rank);
51: MPI_Comm_size(mat->comm, &size);
52: if (part->n != size) {
53: SETERRQ(PETSC_ERR_SUP, "Supports exactly one domain per processor");
54: }
56: /* convert adjacency matrix to MPIAdj if needed*/
57: PetscTypeCompare((PetscObject) mat, MATMPIADJ, &flg);
58: if (!flg) {
59: MatConvert(mat, MATMPIADJ, MAT_INITIAL_MATRIX, &matMPI);
60: } else
61: matMPI = mat;
63: adj = (Mat_MPIAdj *) matMPI->data; /* adj contains adjacency graph */
64: {
65: /* definition of Jostle library arguments */
66: int nnodes = matMPI->M; /* number of vertices in full graph */
67: int offset = 0; /* 0 for C array indexing */
68: int core = matMPI->m;
69: int halo = 0; /* obsolete with contiguous format */
70: int *index_jostle; /* contribution of each processor */
71: int nparts = part->n;
72: int *part_wt = NULL;
74: int *partition; /* set number of each vtx (length n) */
75: int *degree; /* degree for each core nodes */
76: int *edges = adj->j;
77: int *node_wt = NULL; /* nodes weights */
78: int *edge_wt = NULL; /* edges weights */
79: double *coords = NULL; /* not used (cf jostle documentation) */
81: int local_nedges = adj->nz;
82: int dimension = 0; /* not used */
83: int output_level = jostle_struct->output;
84: char env_str[256];
86: /* allocate index_jostle */
87: PetscMalloc(nparts * sizeof(int), &index_jostle);
89: /* compute number of core nodes for each one */
90: for (i = 0; i < nparts - 1; i++)
91: index_jostle[i] = adj->rowners[i + 1] - adj->rowners[i];
92: index_jostle[nparts - 1] = nnodes - adj->rowners[nparts - 1];
94: /* allocate the partition vector */
95: PetscMalloc(core * sizeof(int), &partition);
97: /* build the degree vector and the local_nedges value */
98: PetscMalloc(core * sizeof(int), °ree);
99: for (i = 0; i < core; i++)
100: degree[i] = adj->i[i + 1] - adj->i[i];
102: /* library call */
103: pjostle_init(&size, &rank);
104: pjostle_comm(&matMPI->comm);
105: jostle_env("format = contiguous");
106: jostle_env("timer = off");
108: sprintf(env_str, "threshold = %d", jostle_struct->nbvtxcoarsed);
109: jostle_env(env_str);
111: if (jostle_struct->coarse_seq)
112: jostle_env("matching = local");
114: /* redirect output */
115: #ifdef PETSC_HAVE_UNISTD_H
116: fd_stdout = dup(1);
117: pipe(fd_pipe);
118: close(1);
119: dup2(fd_pipe[1], 1);
120: #endif
122: pjostle(&nnodes, &offset, &core, &halo, index_jostle, degree, node_wt,
123: partition, &local_nedges, edges, edge_wt, &nparts,
124: part_wt, &output_level, &dimension, coords);
126: printf("Jostle Partitioner statistics\ncut : %d, balance : %f, runtime : %f, mem used : %d\n",
127: jostle_cut(), jostle_bal(), jostle_tim(), jostle_mem());
129: #ifdef PETSC_HAVE_UNISTD_H
130: PetscMalloc(SIZE_LOG * sizeof(char), &(jostle_struct->mesg_log));
131: fflush(stdout);
132: count = read(fd_pipe[0], jostle_struct->mesg_log, (SIZE_LOG - 1) * sizeof(char));
133: if (count < 0)
134: count = 0;
135: jostle_struct->mesg_log[count] = 0;
136: close(1);
137: dup2(fd_stdout, 1);
138: close(fd_stdout);
139: close(fd_pipe[0]);
140: close(fd_pipe[1]);
141: #endif
143: /* We free the memory used by jostle */
144: PetscFree(index_jostle);
145: PetscFree(degree);
147: /* Creation of the index set */
148: ISCreateGeneral(part->comm, mat->m, partition, partitioning);
150: if (matMPI != mat) {
151: MatDestroy(matMPI);
152: }
154: PetscFree(partition);
155: }
157: return(0);
158: }
163: PetscErrorCode MatPartitioningView_Jostle(MatPartitioning part, PetscViewer viewer)
164: {
165: MatPartitioning_Jostle *jostle_struct = (MatPartitioning_Jostle *) part->data;
166: PetscErrorCode ierr;
167: PetscTruth iascii;
170: PetscTypeCompare((PetscObject) viewer, PETSC_VIEWER_ASCII, &iascii);
171: if (iascii) {
172: if (jostle_struct->mesg_log) {
173: PetscViewerASCIIPrintf(viewer, "%s\n", jostle_struct->mesg_log);
174: }
175: } else {
176: SETERRQ1(PETSC_ERR_SUP, "Viewer type %s not supported for this Jostle partitioner",((PetscObject) viewer)->type_name);
177: }
178: return(0);
179: }
183: /*@
184: MatPartitioningJostleSetCoarseLevel - Set the coarse level
185:
186: Input Parameter:
187: . part - the partitioning context
188: . level - the coarse level in range [0.0,1.0]
190: Level: advanced
192: @*/
193: PetscErrorCode MatPartitioningJostleSetCoarseLevel(MatPartitioning part, PetscReal level)
194: {
195: MatPartitioning_Jostle *jostle_struct = (MatPartitioning_Jostle *) part->data;
199: if (level < 0.0 || level > 1.0) {
200: SETERRQ(PETSC_ERR_ARG_OUTOFRANGE,
201: "Jostle: level of coarsening out of range [0.0-1.0]");
202: } else
203: jostle_struct->nbvtxcoarsed = (int)(part->adj->N * level);
205: if (jostle_struct->nbvtxcoarsed < 20)
206: jostle_struct->nbvtxcoarsed = 20;
208: return(0);
209: }
213: /*@
214: MatPartitioningJostleSetCoarseSequential - Use the sequential code to
215: do the partitioning of the coarse grid.
217: Input Parameter:
218: . part - the partitioning context
220: Level: advanced
222: @*/
223: PetscErrorCode MatPartitioningJostleSetCoarseSequential(MatPartitioning part)
224: {
225: MatPartitioning_Jostle *jostle_struct =
226: (MatPartitioning_Jostle *) part->data;
228: jostle_struct->coarse_seq = 1;
229: return(0);
230: }
234: PetscErrorCode MatPartitioningSetFromOptions_Jostle(MatPartitioning part)
235: {
237: PetscTruth flag;
238: PetscReal level;
241: PetscOptionsHead("Set Jostle partitioning options");
243: PetscOptionsReal("-mat_partitioning_jostle_coarse_level",
244: "Coarse level", "MatPartitioningJostleSetCoarseLevel", 0,
245: &level, &flag);
246: if (flag)
247: MatPartitioningJostleSetCoarseLevel(part, level);
249: PetscOptionsName("-mat_partitioning_jostle_coarse_sequential",
250: "Use sequential coarse partitioner",
251: "MatPartitioningJostleSetCoarseSequential", &flag);
252: if (flag) {
253: MatPartitioningJostleSetCoarseSequential(part);
254: }
256: PetscOptionsTail();
257: return(0);
258: }
263: PetscErrorCode MatPartitioningDestroy_Jostle(MatPartitioning part)
264: {
265: MatPartitioning_Jostle *jostle_struct = (MatPartitioning_Jostle *) part->data;
266: PetscErrorCode ierr;
269: PetscFree(jostle_struct->mesg_log);
270: PetscFree(jostle_struct);
271: return(0);
272: }
277: /*@C
278: MAT_PARTITIONING_JOSTLE - Creates a partitioning context via the external package Jostle.
280: Collective on MPI_Comm
282: Input Parameter:
283: . part - the partitioning context
285: Options Database Keys:
286: + -mat_partitioning_jostle_coarse_level <0>: Coarse level (MatPartitioningJostleSetCoarseLevel)
287: - -mat_partitioning_jostle_coarse_sequential: Use sequential coarse partitioner (MatPartitioningJostleSetCoarseSequential)
289: Level: beginner
291: Notes: See http://www.gre.ac.uk/~c.walshaw/jostle/
293: .keywords: Partitioning, create, context
295: .seealso: MatPartitioningSetType(), MatPartitioningType
297: @*/
298: PetscErrorCode MatPartitioningCreate_Jostle(MatPartitioning part)
299: {
301: MatPartitioning_Jostle *jostle_struct;
304: PetscNew(MatPartitioning_Jostle, &jostle_struct);
306: jostle_struct->nbvtxcoarsed = 20;
307: jostle_struct->output = 0;
308: jostle_struct->coarse_seq = 0;
309: jostle_struct->mesg_log = NULL;
311: part->ops->apply = MatPartitioningApply_Jostle;
312: part->ops->view = MatPartitioningView_Jostle;
313: part->ops->destroy = MatPartitioningDestroy_Jostle;
314: part->ops->setfromoptions = MatPartitioningSetFromOptions_Jostle;
315: part->data = (void*) jostle_struct;
317: return(0);
318: }