]>
git.saurik.com Git - apt.git/blob - ftparchive/writer.cc
1 // -*- mode: cpp; mode: fold -*-
3 // $Id: writer.cc,v 1.2 2001/02/20 07:03:18 jgg Exp $
4 /* ######################################################################
8 The file writer classes. These write various types of output, sources,
11 ##################################################################### */
13 // Include Files /*{{{*/
15 #pragma implementation "writer.h"
20 #include <apt-pkg/strutl.h>
21 #include <apt-pkg/error.h>
22 #include <apt-pkg/configuration.h>
23 #include <apt-pkg/md5.h>
24 #include <apt-pkg/deblistparser.h>
26 #include <sys/types.h>
31 #include "apt-ftparchive.h"
32 #include "multicompress.h"
35 FTWScanner
*FTWScanner::Owner
;
37 // FTWScanner::FTWScanner - Constructor /*{{{*/
38 // ---------------------------------------------------------------------
40 FTWScanner::FTWScanner()
43 NoLinkAct
= !_config
->FindB("APT::FTPArchive::DeLinkAct",true);
47 long PMax
= pathconf(".",_PC_PATH_MAX
);
49 RealPath
= new char[PMax
];
52 // FTWScanner::Scanner - FTW Scanner /*{{{*/
53 // ---------------------------------------------------------------------
54 /* This is the FTW scanner, it processes each directory element in the
56 int FTWScanner::Scanner(const char *File
,const struct stat
*sb
,int Flag
)
61 c1out
<< "W: Unable to read directory " << File
<< endl
;
66 c1out
<< "W: Unable to stat " << File
<< endl
;
71 // See if it is a .deb
76 for (; Owner
->Ext
[CurExt
] != 0; CurExt
++)
77 if (strcmp(File
+strlen(File
)-strlen(Owner
->Ext
[CurExt
]),
78 Owner
->Ext
[CurExt
]) == 0)
80 if (Owner
->Ext
[CurExt
] == 0)
83 /* Process it. If the file is a link then resolve it into an absolute
84 name.. This works best if the directory components the scanner are
85 given are not links themselves. */
87 Owner
->OriginalPath
= File
;
88 if (Owner
->RealPath
!= 0 && readlink(File
,Jnk
,sizeof(Jnk
)) != -1 &&
89 realpath(File
,Owner
->RealPath
) != 0)
90 Owner
->DoPackage(Owner
->RealPath
);
92 Owner
->DoPackage(File
);
94 if (_error
->empty() == false)
96 // Print any errors or warnings found
98 bool SeenPath
= false;
99 while (_error
->empty() == false)
103 bool Type
= _error
->PopMessage(Err
);
105 c1out
<< "E: " << Err
<< endl
;
107 c1out
<< "W: " << Err
<< endl
;
109 if (Err
.find(File
) != string::npos
)
113 if (SeenPath
== false)
114 cerr
<< "E: Errors apply to file '" << File
<< "'" << endl
;
121 // FTWScanner::RecursiveScan - Just scan a directory tree /*{{{*/
122 // ---------------------------------------------------------------------
124 bool FTWScanner::RecursiveScan(string Dir
)
126 /* If noprefix is set then jam the scan root in, so we don't generate
127 link followed paths out of control */
128 if (InternalPrefix
.empty() == true)
130 if (realpath(Dir
.c_str(),RealPath
) == 0)
131 return _error
->Errno("realpath","Failed to resolve %s",Dir
.c_str());
132 InternalPrefix
= RealPath
;
135 // Do recursive directory searching
137 int Res
= ftw(Dir
.c_str(),Scanner
,30);
139 // Error treewalking?
142 if (_error
->PendingError() == false)
143 _error
->Errno("ftw","Tree walking failed");
150 // FTWScanner::LoadFileList - Load the file list from a file /*{{{*/
151 // ---------------------------------------------------------------------
152 /* This is an alternative to using FTW to locate files, it reads the list
153 of files from another file. */
154 bool FTWScanner::LoadFileList(string Dir
,string File
)
156 /* If noprefix is set then jam the scan root in, so we don't generate
157 link followed paths out of control */
158 if (InternalPrefix
.empty() == true)
160 if (realpath(Dir
.c_str(),RealPath
) == 0)
161 return _error
->Errno("realpath","Failed to resolve %s",Dir
.c_str());
162 InternalPrefix
= RealPath
;
166 FILE *List
= fopen(File
.c_str(),"r");
168 return _error
->Errno("fopen","Failed to open %s",File
.c_str());
170 /* We are a tad tricky here.. We prefix the buffer with the directory
171 name, that way if we need a full path with just use line.. Sneaky and
175 if (Dir
.empty() == true || Dir
.end()[-1] != '/')
176 FileStart
= Line
+ snprintf(Line
,sizeof(Line
),"%s/",Dir
.c_str());
178 FileStart
= Line
+ snprintf(Line
,sizeof(Line
),"%s",Dir
.c_str());
179 while (fgets(FileStart
,sizeof(Line
) - (FileStart
- Line
),List
) != 0)
181 char *FileName
= _strstrip(FileStart
);
182 if (FileName
[0] == 0)
185 if (FileName
[0] != '/')
187 if (FileName
!= FileStart
)
188 memmove(FileStart
,FileName
,strlen(FileStart
));
194 if (stat(FileName
,&St
) != 0)
197 if (Scanner(FileName
,&St
,Flag
) != 0)
205 // FTWScanner::Delink - Delink symlinks /*{{{*/
206 // ---------------------------------------------------------------------
208 bool FTWScanner::Delink(string
&FileName
,const char *OriginalPath
,
209 unsigned long &DeLinkBytes
,
212 // See if this isn't an internaly prefix'd file name.
213 if (InternalPrefix
.empty() == false &&
214 InternalPrefix
.length() < FileName
.length() &&
215 stringcmp(FileName
.begin(),FileName
.begin() + InternalPrefix
.length(),
216 InternalPrefix
.begin(),InternalPrefix
.end()) != 0)
218 if (DeLinkLimit
!= 0 && DeLinkBytes
/1024 < DeLinkLimit
)
220 // Tidy up the display
221 if (DeLinkBytes
== 0)
225 c1out
<< " DeLink " << (OriginalPath
+ InternalPrefix
.length())
226 << " [" << SizeToStr(St
.st_size
) << "B]" << endl
<< flush
;
228 if (NoLinkAct
== false)
231 if (readlink(OriginalPath
,OldLink
,sizeof(OldLink
)) == -1)
232 _error
->Errno("readlink","Failed to readlink %s",OriginalPath
);
235 if (unlink(OriginalPath
) != 0)
236 _error
->Errno("unlink","Failed to unlink %s",OriginalPath
);
239 if (link(FileName
.c_str(),OriginalPath
) != 0)
241 // Panic! Restore the symlink
242 symlink(OldLink
,OriginalPath
);
243 return _error
->Errno("link","*** Failed to link %s to %s",
251 DeLinkBytes
+= St
.st_size
;
252 if (DeLinkBytes
/1024 >= DeLinkLimit
)
253 c1out
<< " DeLink limit of " << SizeToStr(DeLinkBytes
) << "B hit." << endl
;
256 FileName
= OriginalPath
;
262 // FTWScanner::SetExts - Set extensions to support /*{{{*/
263 // ---------------------------------------------------------------------
265 bool FTWScanner::SetExts(string Vals
)
268 TmpExt
= new char[Vals
.length()+1];
269 strcpy(TmpExt
,Vals
.c_str());
270 return TokSplitString(' ',TmpExt
,(char **)Ext
,sizeof(Ext
)/sizeof(Ext
[0]));
274 // PackagesWriter::PackagesWriter - Constructor /*{{{*/
275 // ---------------------------------------------------------------------
277 PackagesWriter::PackagesWriter(string DB
,string Overrides
) :
278 Db(DB
),Stats(Db
.Stats
)
285 // Process the command line options
286 DoMD5
= _config
->FindB("APT::FTPArchive::MD5",true);
287 DoContents
= _config
->FindB("APT::FTPArchive::Contents",true);
288 NoOverride
= _config
->FindB("APT::FTPArchive::NoOverrideMsg",false);
290 if (Db
.Loaded() == false)
293 // Read the override file
294 if (Overrides
.empty() == false && Over
.ReadOverride(Overrides
) == false)
298 _error
->DumpErrors();
301 // PackagesWriter::DoPackage - Process a single package /*{{{*/
302 // ---------------------------------------------------------------------
303 /* This method takes a package and gets its control information and
304 MD5 then writes out a control record with the proper fields rewritten
305 and the path/size/hash appended. */
306 bool PackagesWriter::DoPackage(string FileName
)
309 FileFd
F(FileName
,FileFd::ReadOnly
);
310 if (_error
->PendingError() == true)
313 // Stat the file for later
315 if (fstat(F
.Fd(),&St
) != 0)
316 return _error
->Errno("fstat","Failed to stat %s",FileName
.c_str());
318 // Pull all the data we need form the DB
320 if (Db
.SetFile(FileName
,St
,&F
) == false ||
321 Db
.LoadControl() == false ||
322 (DoContents
== true && Db
.LoadContents(true) == false) ||
323 (DoMD5
== true && Db
.GetMD5(MD5Res
,false) == false))
326 if (Delink(FileName
,OriginalPath
,Stats
.DeLinkBytes
,St
) == false)
329 // Lookup the overide information
330 pkgTagSection
&Tags
= Db
.Control
.Section
;
331 string Package
= Tags
.FindS("Package");
333 Override::Item
*OverItem
= Over
.GetItem(Package
);
335 if (Package
.empty() == true)
336 return _error
->Error("Archive had no package field");
338 // If we need to do any rewriting of the header do it now..
341 if (NoOverride
== false)
344 c1out
<< " " << Package
<< " has no override entry" << endl
;
348 Tmp
.Section
= Tags
.FindS("Section");
349 Tmp
.Priority
= Tags
.FindS("Priority");
353 sprintf(Size
,"%lu",St
.st_size
);
355 // Strip the DirStrip prefix from the FileName and add the PathPrefix
357 if (DirStrip
.empty() == false &&
358 FileName
.length() > DirStrip
.length() &&
359 stringcmp(FileName
.begin(),FileName
.begin() + DirStrip
.length(),
360 DirStrip
.begin(),DirStrip
.end()) == 0)
361 NewFileName
= string(FileName
.begin() + DirStrip
.length(),FileName
.end());
363 NewFileName
= FileName
;
364 if (PathPrefix
.empty() == false)
365 NewFileName
= flCombine(PathPrefix
,NewFileName
);
367 // This lists all the changes to the fields we are going to make.
368 TFRewriteData Changes
[] = {{"Size",Size
},
369 {"MD5sum",MD5Res
.c_str()},
370 {"Filename",NewFileName
.c_str()},
371 {"Section",OverItem
->Section
.c_str()},
372 {"Priority",OverItem
->Priority
.c_str()},
375 {}, // For maintainer
378 unsigned int End
= 0;
379 for (End
= 0; Changes
[End
].Tag
!= 0; End
++);
381 // Rewrite the maintainer field if necessary
383 string NewMaint
= OverItem
->SwapMaint(Tags
.FindS("Maintainer"),MaintFailed
);
384 if (MaintFailed
== true)
386 if (NoOverride
== false)
389 c1out
<< " " << Package
<< " maintainer is " <<
390 Tags
.FindS("Maintainer") << " not " <<
391 OverItem
->OldMaint
<< endl
;
395 if (NewMaint
.empty() == false)
397 Changes
[End
].Rewrite
= NewMaint
.c_str();
398 Changes
[End
++].Tag
= "Maintainer";
401 /* Get rid of the Optional tag. This is an ugly, ugly, ugly hack that
402 dpkg-scanpackages does.. Well sort of. dpkg-scanpackages just does renaming
403 but dpkg does this append bit. So we do the append bit, at least that way the
404 status file and package file will remain similar. There are other transforms
405 but optional is the only legacy one still in use for some lazy reason. */
406 string OptionalStr
= Tags
.FindS("Optional");
407 if (OptionalStr
.empty() == false)
409 if (Tags
.FindS("Suggests").empty() == false)
410 OptionalStr
= Tags
.FindS("Suggests") + ", " + OptionalStr
;
411 Changes
[End
].Rewrite
= OptionalStr
.c_str();
412 Changes
[End
++].Tag
= "Suggests";
415 // Rewrite and store the fields.
416 if (TFRewrite(Output
,Tags
,TFRewritePackageOrder
,Changes
) == false)
418 fprintf(Output
,"\n");
424 // SourcesWriter::SourcesWriter - Constructor /*{{{*/
425 // ---------------------------------------------------------------------
427 SourcesWriter::SourcesWriter(string BOverrides
,string SOverrides
)
436 // Process the command line options
437 NoOverride
= _config
->FindB("APT::FTPArchive::NoOverrideMsg",false);
439 // Read the override file
440 if (BOverrides
.empty() == false && BOver
.ReadOverride(BOverrides
) == false)
445 if (SOverrides
.empty() == false && FileExists(SOverrides
) == true &&
446 SOver
.ReadOverride(SOverrides
,true) == false)
448 // _error->DumpErrors();
451 // SourcesWriter::DoPackage - Process a single package /*{{{*/
452 // ---------------------------------------------------------------------
454 bool SourcesWriter::DoPackage(string FileName
)
457 FileFd
F(FileName
,FileFd::ReadOnly
);
458 if (_error
->PendingError() == true)
461 // Stat the file for later
463 if (fstat(F
.Fd(),&St
) != 0)
464 return _error
->Errno("fstat","Failed to stat %s",FileName
.c_str());
466 if (St
.st_size
> 128*1024)
467 return _error
->Error("DSC file '%s' is too large!",FileName
.c_str());
469 if (BufSize
< (unsigned)St
.st_size
+1)
471 BufSize
= St
.st_size
+1;
472 Buffer
= (char *)realloc(Buffer
,St
.st_size
+1);
475 if (F
.Read(Buffer
,St
.st_size
) == false)
479 char *Start
= Buffer
;
480 char *BlkEnd
= Buffer
+ St
.st_size
;
482 MD5
.Add((unsigned char *)Start
,BlkEnd
- Start
);
484 // Add an extra \n to the end, just in case
487 /* Remove the PGP trailer. Some .dsc's have this without a blank line
489 const char *Key
= "-----BEGIN PGP SIGNATURE-----";
490 for (char *MsgEnd
= Start
; MsgEnd
< BlkEnd
- strlen(Key
) -1; MsgEnd
++)
492 if (*MsgEnd
== '\n' && strncmp(MsgEnd
+1,Key
,strlen(Key
)) == 0)
499 /* Read records until we locate the Source record. This neatly skips the
500 GPG header (which is RFC822 formed) without any trouble. */
505 if (Tags
.Scan(Start
,BlkEnd
- Start
) == false)
506 return _error
->Error("Could not find a record in the DSC '%s'",FileName
.c_str());
507 if (Tags
.Find("Source",Pos
) == true)
509 Start
+= Tags
.size();
514 // Lookup the overide information, finding first the best priority.
517 string Bins
= Tags
.FindS("Binary");
518 Override::Item
*OverItem
= 0;
519 if (Bins
.empty() == false && Bins
.length() < sizeof(Buffer
))
521 strcpy(Buffer
,Bins
.c_str());
523 // Ignore too-long errors.
525 TokSplitString(',',Buffer
,BinList
,sizeof(BinList
)/sizeof(BinList
[0]));
527 // Look at all the binaries
528 unsigned char BestPrioV
= pkgCache::State::Extra
;
529 for (unsigned I
= 0; BinList
[I
] != 0; I
++)
531 Override::Item
*Itm
= BOver
.GetItem(BinList
[I
]);
537 unsigned char NewPrioV
= debListParser::GetPrio(Itm
->Priority
);
538 if (NewPrioV
< BestPrioV
|| BestPrio
.empty() == true)
540 BestPrioV
= NewPrioV
;
541 BestPrio
= Itm
->Priority
;
546 // If we need to do any rewriting of the header do it now..
550 if (NoOverride
== false)
553 c1out
<< " " << Tags
.FindS("Source") << " has no override entry" << endl
;
559 Override::Item
*SOverItem
= SOver
.GetItem(Tags
.FindS("Source"));
562 SOverItem
= BOver
.GetItem(Tags
.FindS("Source"));
564 SOverItem
= OverItem
;
567 // Add the dsc to the files hash list
569 snprintf(Files
,sizeof(Files
),"\n %s %lu %s\n %s",
570 string(MD5
.Result()).c_str(),St
.st_size
,
571 flNotDir(FileName
).c_str(),
572 Tags
.FindS("Files").c_str());
574 // Strip the DirStrip prefix from the FileName and add the PathPrefix
576 if (DirStrip
.empty() == false &&
577 FileName
.length() > DirStrip
.length() &&
578 stringcmp(OriginalPath
,OriginalPath
+ DirStrip
.length(),
579 DirStrip
.begin(),DirStrip
.end()) == 0)
580 NewFileName
= string(OriginalPath
+ DirStrip
.length());
582 NewFileName
= OriginalPath
;
583 if (PathPrefix
.empty() == false)
584 NewFileName
= flCombine(PathPrefix
,NewFileName
);
586 string Directory
= flNotFile(OriginalPath
);
587 string Package
= Tags
.FindS("Source");
589 // Perform the delinking operation over all of the files
591 const char *C
= Files
;
592 for (;isspace(*C
); C
++);
595 // Parse each of the elements
596 if (ParseQuoteWord(C
,ParseJnk
) == false ||
597 ParseQuoteWord(C
,ParseJnk
) == false ||
598 ParseQuoteWord(C
,ParseJnk
) == false)
599 return _error
->Error("Error parsing file record");
602 string OriginalPath
= Directory
+ ParseJnk
;
603 if (RealPath
!= 0 && readlink(OriginalPath
.c_str(),Jnk
,sizeof(Jnk
)) != -1 &&
604 realpath(OriginalPath
.c_str(),RealPath
) != 0)
606 string RP
= RealPath
;
607 if (Delink(RP
,OriginalPath
.c_str(),Stats
.DeLinkBytes
,St
) == false)
612 Directory
= flNotFile(NewFileName
);
613 if (Directory
.length() > 2)
614 Directory
.erase(Directory
.end()-1);
616 // This lists all the changes to the fields we are going to make.
617 TFRewriteData Changes
[] = {{"Source",Package
.c_str(),"Package"},
619 {"Directory",Directory
.c_str()},
620 {"Section",SOverItem
->Section
.c_str()},
621 {"Priority",BestPrio
.c_str()},
623 {}, // For maintainer
625 unsigned int End
= 0;
626 for (End
= 0; Changes
[End
].Tag
!= 0; End
++);
628 // Rewrite the maintainer field if necessary
630 string NewMaint
= OverItem
->SwapMaint(Tags
.FindS("Maintainer"),MaintFailed
);
631 if (MaintFailed
== true)
633 if (NoOverride
== false)
636 c1out
<< " " << Package
<< " maintainer is " <<
637 Tags
.FindS("Maintainer") << " not " <<
638 OverItem
->OldMaint
<< endl
;
641 if (NewMaint
.empty() == false)
643 Changes
[End
].Rewrite
= NewMaint
.c_str();
644 Changes
[End
++].Tag
= "Maintainer";
647 // Rewrite and store the fields.
648 if (TFRewrite(Output
,Tags
,TFRewriteSourceOrder
,Changes
) == false)
650 fprintf(Output
,"\n");
658 // ContentsWriter::ContentsWriter - Constructor /*{{{*/
659 // ---------------------------------------------------------------------
661 ContentsWriter::ContentsWriter(string DB
) :
662 Db(DB
), Stats(Db
.Stats
)
670 // ContentsWriter::DoPackage - Process a single package /*{{{*/
671 // ---------------------------------------------------------------------
672 /* If Package is the empty string the control record will be parsed to
673 determine what the package name is. */
674 bool ContentsWriter::DoPackage(string FileName
,string Package
)
677 FileFd
F(FileName
,FileFd::ReadOnly
);
678 if (_error
->PendingError() == true)
681 // Stat the file for later
683 if (fstat(F
.Fd(),&St
) != 0)
684 return _error
->Errno("fstat","Failed too stat %s",FileName
.c_str());
687 if (Db
.SetFile(FileName
,St
,&F
) == false ||
688 Db
.LoadContents(false) == false)
691 // Parse the package name
692 if (Package
.empty() == true)
694 if (Db
.LoadControl() == false)
696 Package
= Db
.Control
.Section
.FindS("Package");
699 Db
.Contents
.Add(Gen
,Package
);
704 // ContentsWriter::ReadFromPkgs - Read from a packages file /*{{{*/
705 // ---------------------------------------------------------------------
707 bool ContentsWriter::ReadFromPkgs(string PkgFile
,string PkgCompress
)
709 MultiCompress
Pkgs(PkgFile
,PkgCompress
,0,false);
710 if (_error
->PendingError() == true)
713 // Open the package file
716 if (Pkgs
.OpenOld(CompFd
,Proc
) == false)
720 FileFd
Fd(CompFd
,false);
721 pkgTagFile
Tags(&Fd
);
722 if (_error
->PendingError() == true)
724 Pkgs
.CloseOld(CompFd
,Proc
);
729 pkgTagSection Section
;
730 while (Tags
.Step(Section
) == true)
732 string File
= flCombine(Prefix
,Section
.FindS("FileName"));
733 string Package
= Section
.FindS("Section");
734 if (Package
.empty() == false && Package
.end()[-1] != '/')
737 Package
+= Section
.FindS("Package");
740 Package
+= Section
.FindS("Package");
742 DoPackage(File
,Package
);
743 if (_error
->empty() == false)
745 _error
->Error("Errors apply to file '%s'",File
.c_str());
746 _error
->DumpErrors();
750 // Tidy the compressor
751 if (Pkgs
.CloseOld(CompFd
,Proc
) == false)