#include <stdlib.h>
#include <stdio.h>
#include <math.h>
#include <assert.h>
#include <mpi.h>
#include "smut.h"
#include "stripesmut.h"
int
main (int argc, char **argv)
{
FILE *f;
int err;
struct spcsr_t A, AT;
struct gcsr_t gA, gAT;
double timein, timeout;
int rank, csize;
int p;
assert(sizeof(int) == 4);
spcsr_init_clear(&A);
gcsr_init_clear(&gA);
gcsr_init_clear(&gAT);
MPI_Init (&argc, &argv);
MPI_Comm_size (MPI_COMM_WORLD, &csize);
MPI_Comm_rank (MPI_COMM_WORLD, &rank);
if (0 == rank) {
printf ("MPI world size: %d\n", csize);
if (argc <= 2) {
printf("No files\n");
return -1;
}
f = fopen(argv[1], "rb");
if (!f) {
perror("Error opening file: ");
return -1;
}
err = spcsr_load_binfile (f, &A);
if (err) {
printf ("Error reading: %d\n", err);
return -1;
}
fclose(f);
spcsr_transpose (&A, &AT);
}
gcsr_take_csr_root (&A, &gA, MPI_COMM_WORLD);
{
for (p = 0; p < csize; ++p) {
MPI_Barrier(MPI_COMM_WORLD);
if (p == rank) {
printf("**** %d: gA(%d, %d), lA(%d, %d; %d) rows [%d, %d)\n",
rank, gA.nr, gA.nc, gA.local.nr, gA.local.nc, gA.local.nent,
gA.rowmap.lindstart, gA.rowmap.lindstart + gA.rowmap.lnind);
printf("**** %d: local rowoff[1] = %d\n", rank,
(gA.local.rowoff? gA.local.rowoff[1] : -1));
}
MPI_Barrier(MPI_COMM_WORLD);
}
}
/* fprintf(stderr, "%d: ugh1\n", rank); */
gcsr_redist_ents (&gA, 0, MPI_COMM_WORLD);
/* fprintf(stderr, "%d: ugh2\n", rank); */
{
for (p = 0; p < csize; ++p) {
MPI_Barrier(MPI_COMM_WORLD);
if (p == rank) {
printf("---- %d: gA(%d, %d), lA(%d, %d; %d) rows [%d, %d)\n",
rank, gA.nr, gA.nc, gA.local.nr, gA.local.nc, gA.local.nent,
gA.rowmap.lindstart, gA.rowmap.lindstart + gA.rowmap.lnind);
printf("---- %d: local rowoff[1] = %d\n", rank,
(gA.local.rowoff? gA.local.rowoff[1] : -1));
}
MPI_Barrier(MPI_COMM_WORLD);
}
}
timein = MPI_Wtime();
gcsr_transpose_onroot (&gA, &gAT, NULL, MPI_COMM_WORLD);
timeout = MPI_Wtime();
{
for (p = 0; p < csize; ++p) {
MPI_Barrier(MPI_COMM_WORLD);
if (p == rank) {
printf("++++ %d: gA(%d, %d), lA(%d, %d; %d) rows [%d, %d)\n",
rank, gA.nr, gA.nc, gA.local.nr, gA.local.nc, gA.local.nent,
gA.rowmap.lindstart, gA.rowmap.lindstart + gA.rowmap.lnind);
printf("++++ %d: local rowoff[1] = %d\n", rank,
(gA.local.rowoff? gA.local.rowoff[1] : -1));
}
MPI_Barrier(MPI_COMM_WORLD);
}
}
{
for (p = 0; p < csize; ++p) {
MPI_Barrier(MPI_COMM_WORLD);
if (p == rank) {
printf("++__ %d: gAT(%d, %d), lA(%d, %d; %d) rows [%d, %d)\n",
rank, gAT.nr, gAT.nc, gAT.local.nr, gAT.local.nc,
gAT.local.nent,
gAT.rowmap.lindstart, gAT.rowmap.lindstart + gAT.rowmap.lnind);
printf("++__ %d: local rowoff[1] = %d\n", rank,
(gAT.local.rowoff? gAT.local.rowoff[1] : -1));
}
MPI_Barrier(MPI_COMM_WORLD);
}
}
{
int i, j, k;
int *seenr, *seenc;
int *rootseenr = NULL, *rootseenc = NULL;
seenr = calloc (gAT.nr + gAT.nc, sizeof(int));
seenc = seenr + gAT.nr;
if (0 == rank) {
rootseenr = calloc (gAT.nr + gAT.nc, sizeof(int));
rootseenc = rootseenr + gAT.nr;
}
for (i = 0; i < gAT.local.nr; ++i) {
const int gi = i + gAT.rowmap.lindstart;
if (gAT.local.rowoff[i] != gAT.local.rowoff[i+1])
seenr[gi] = 1;
for (k = gAT.local.rowoff[i]; k < gAT.local.rowoff[i+1]; ++k) {
j = gAT.local.colind[k];
if (gAT.colmap.l2g)
seenc[gAT.colmap.l2g[j]] = 1;
else
seenc[j] = 1;
}
}
MPI_Reduce (seenr, rootseenr, gAT.nr + gAT.nc, MPI_INT, MPI_MAX, 0,
MPI_COMM_WORLD);
if (0 == rank) {
for (i = 0; i < gAT.nr; ++i)
if (!rootseenr[i])
fprintf (stderr, "%d:: no one sees row %d\n", rank, i);
for (j = 0; j < gAT.nc; ++j)
if (!rootseenc[j])
fprintf (stderr, "%d:: no one sees col %d\n", rank, j);
}
free(seenr);
if (rootseenr) free(rootseenr);
}
gcsr_give_csr_root (&gAT, &A, MPI_COMM_WORLD);
if (0 == rank) {
int i, j, k;
int *seenr, *seenc;
seenr = calloc (A.nr + A.nc, sizeof(int));
seenc = seenr + A.nr;
for (i = 0; i < A.nr; ++i) {
if (A.rowoff[i] != A.rowoff[i+1])
seenr[i] = 1;
for (k = A.rowoff[i]; k < A.rowoff[i+1]; ++k) {
j = A.colind[k];
seenc[j] = 1;
}
}
for (i = 0; i < A.nr; ++i)
if (!seenr[i])
fprintf (stderr, "%d:: row %d not in root AT\n", rank, i);
for (j = 0; j < A.nc; ++j)
if (!seenc[j])
fprintf (stderr, "%d:: col %d not in root AT\n", rank, j);
free(seenr);
}
if (0 == rank) {
int i, k;
f = fopen(argv[2], "wb");
if (!f) {
perror("Error opening file: ");
return -1;
}
err = spcsr_write_binfile (&A, f);
fclose(f);
assert(A.nr == AT.nr);
assert(A.nc == AT.nc);
assert(A.nent == AT.nent);
for (i = 0; i < A.nr; ++i) {
if (A.rowoff[i] != AT.rowoff[i]) {
fprintf(stderr, "A and copy differ, rowoff[%d] = %d or %d\n",
i, A.rowoff[i], AT.rowoff[i]);
}
for (k = A.rowoff[i]; k != A.rowoff[i+1]; ++k) {
if (A.colind[k] != AT.colind[k])
fprintf(stderr, "A and copy differ, colind[%d] = %d or %d\n",
k, A.colind[k], AT.colind[k]);
if (A.entry[k] != AT.entry[k])
fprintf(stderr, "A and copy differ, entry[%d] = %g or %g\n",
k, A.entry[k], AT.entry[k]);
}
}
}
timeout -= timein;
if (0 != rank)
MPI_Gather (&timeout, 1, MPI_DOUBLE,
NULL, 1, MPI_DOUBLE,
0, MPI_COMM_WORLD);
else {
int p;
double *times;
long double tottime = 0.0L;
double mintime = HUGE_VAL, maxtime = 0.0;
times = malloc (csize * sizeof(double));
MPI_Gather (&timeout, 1, MPI_DOUBLE,
times, 1, MPI_DOUBLE,
0, MPI_COMM_WORLD);
printf("Times:");
for (p = 0; p < csize; ++p) {
tottime += times[p];
mintime = (times[p] < mintime? times[p] : mintime);
maxtime = (times[p] > maxtime? times[p] : mintime);
printf(" %g", times[p]);
}
printf("\nmax: %g min: %g tot: %g mean: %g\n",
maxtime, mintime, (double)tottime, (double)(tottime / csize));
free(times);
}
spcsr_free (&A);
gcsr_free (&gA, MPI_COMM_WORLD);
MPI_Finalize();
return 0;
}