]>
git.saurik.com Git - apt.git/blob - ftparchive/writer.cc
1 // -*- mode: cpp; mode: fold -*-
3 // $Id: writer.cc,v 1.3 2001/05/29 04:08:09 jgg Exp $
4 /* ######################################################################
8 The file writer classes. These write various types of output, sources,
11 ##################################################################### */
13 // Include Files /*{{{*/
15 #pragma implementation "writer.h"
20 #include <apt-pkg/strutl.h>
21 #include <apt-pkg/error.h>
22 #include <apt-pkg/configuration.h>
23 #include <apt-pkg/md5.h>
24 #include <apt-pkg/deblistparser.h>
26 #include <sys/types.h>
32 #include "apt-ftparchive.h"
33 #include "multicompress.h"
37 FTWScanner
*FTWScanner::Owner
;
39 // FTWScanner::FTWScanner - Constructor /*{{{*/
40 // ---------------------------------------------------------------------
42 FTWScanner::FTWScanner()
45 NoLinkAct
= !_config
->FindB("APT::FTPArchive::DeLinkAct",true);
49 long PMax
= pathconf(".",_PC_PATH_MAX
);
51 RealPath
= new char[PMax
];
54 // FTWScanner::Scanner - FTW Scanner /*{{{*/
55 // ---------------------------------------------------------------------
56 /* This is the FTW scanner, it processes each directory element in the
58 int FTWScanner::Scanner(const char *File
,const struct stat
*sb
,int Flag
)
63 c1out
<< "W: Unable to read directory " << File
<< endl
;
68 c1out
<< "W: Unable to stat " << File
<< endl
;
73 // See if it is a .deb
78 for (; Owner
->Ext
[CurExt
] != 0; CurExt
++)
79 if (strcmp(File
+strlen(File
)-strlen(Owner
->Ext
[CurExt
]),
80 Owner
->Ext
[CurExt
]) == 0)
82 if (Owner
->Ext
[CurExt
] == 0)
85 /* Process it. If the file is a link then resolve it into an absolute
86 name.. This works best if the directory components the scanner are
87 given are not links themselves. */
89 Owner
->OriginalPath
= File
;
90 if (Owner
->RealPath
!= 0 && readlink(File
,Jnk
,sizeof(Jnk
)) != -1 &&
91 realpath(File
,Owner
->RealPath
) != 0)
92 Owner
->DoPackage(Owner
->RealPath
);
94 Owner
->DoPackage(File
);
96 if (_error
->empty() == false)
98 // Print any errors or warnings found
100 bool SeenPath
= false;
101 while (_error
->empty() == false)
105 bool Type
= _error
->PopMessage(Err
);
107 c1out
<< "E: " << Err
<< endl
;
109 c1out
<< "W: " << Err
<< endl
;
111 if (Err
.find(File
) != string::npos
)
115 if (SeenPath
== false)
116 cerr
<< "E: Errors apply to file '" << File
<< "'" << endl
;
123 // FTWScanner::RecursiveScan - Just scan a directory tree /*{{{*/
124 // ---------------------------------------------------------------------
126 bool FTWScanner::RecursiveScan(string Dir
)
128 /* If noprefix is set then jam the scan root in, so we don't generate
129 link followed paths out of control */
130 if (InternalPrefix
.empty() == true)
132 if (realpath(Dir
.c_str(),RealPath
) == 0)
133 return _error
->Errno("realpath","Failed to resolve %s",Dir
.c_str());
134 InternalPrefix
= RealPath
;
137 // Do recursive directory searching
139 int Res
= ftw(Dir
.c_str(),Scanner
,30);
141 // Error treewalking?
144 if (_error
->PendingError() == false)
145 _error
->Errno("ftw","Tree walking failed");
152 // FTWScanner::LoadFileList - Load the file list from a file /*{{{*/
153 // ---------------------------------------------------------------------
154 /* This is an alternative to using FTW to locate files, it reads the list
155 of files from another file. */
156 bool FTWScanner::LoadFileList(string Dir
,string File
)
158 /* If noprefix is set then jam the scan root in, so we don't generate
159 link followed paths out of control */
160 if (InternalPrefix
.empty() == true)
162 if (realpath(Dir
.c_str(),RealPath
) == 0)
163 return _error
->Errno("realpath","Failed to resolve %s",Dir
.c_str());
164 InternalPrefix
= RealPath
;
168 FILE *List
= fopen(File
.c_str(),"r");
170 return _error
->Errno("fopen","Failed to open %s",File
.c_str());
172 /* We are a tad tricky here.. We prefix the buffer with the directory
173 name, that way if we need a full path with just use line.. Sneaky and
177 if (Dir
.empty() == true || Dir
.end()[-1] != '/')
178 FileStart
= Line
+ snprintf(Line
,sizeof(Line
),"%s/",Dir
.c_str());
180 FileStart
= Line
+ snprintf(Line
,sizeof(Line
),"%s",Dir
.c_str());
181 while (fgets(FileStart
,sizeof(Line
) - (FileStart
- Line
),List
) != 0)
183 char *FileName
= _strstrip(FileStart
);
184 if (FileName
[0] == 0)
187 if (FileName
[0] != '/')
189 if (FileName
!= FileStart
)
190 memmove(FileStart
,FileName
,strlen(FileStart
));
196 if (stat(FileName
,&St
) != 0)
199 if (Scanner(FileName
,&St
,Flag
) != 0)
207 // FTWScanner::Delink - Delink symlinks /*{{{*/
208 // ---------------------------------------------------------------------
210 bool FTWScanner::Delink(string
&FileName
,const char *OriginalPath
,
211 unsigned long &DeLinkBytes
,
214 // See if this isn't an internaly prefix'd file name.
215 if (InternalPrefix
.empty() == false &&
216 InternalPrefix
.length() < FileName
.length() &&
217 stringcmp(FileName
.begin(),FileName
.begin() + InternalPrefix
.length(),
218 InternalPrefix
.begin(),InternalPrefix
.end()) != 0)
220 if (DeLinkLimit
!= 0 && DeLinkBytes
/1024 < DeLinkLimit
)
222 // Tidy up the display
223 if (DeLinkBytes
== 0)
227 c1out
<< " DeLink " << (OriginalPath
+ InternalPrefix
.length())
228 << " [" << SizeToStr(St
.st_size
) << "B]" << endl
<< flush
;
230 if (NoLinkAct
== false)
233 if (readlink(OriginalPath
,OldLink
,sizeof(OldLink
)) == -1)
234 _error
->Errno("readlink","Failed to readlink %s",OriginalPath
);
237 if (unlink(OriginalPath
) != 0)
238 _error
->Errno("unlink","Failed to unlink %s",OriginalPath
);
241 if (link(FileName
.c_str(),OriginalPath
) != 0)
243 // Panic! Restore the symlink
244 symlink(OldLink
,OriginalPath
);
245 return _error
->Errno("link","*** Failed to link %s to %s",
253 DeLinkBytes
+= St
.st_size
;
254 if (DeLinkBytes
/1024 >= DeLinkLimit
)
255 c1out
<< " DeLink limit of " << SizeToStr(DeLinkBytes
) << "B hit." << endl
;
258 FileName
= OriginalPath
;
264 // FTWScanner::SetExts - Set extensions to support /*{{{*/
265 // ---------------------------------------------------------------------
267 bool FTWScanner::SetExts(string Vals
)
270 TmpExt
= new char[Vals
.length()+1];
271 strcpy(TmpExt
,Vals
.c_str());
272 return TokSplitString(' ',TmpExt
,(char **)Ext
,sizeof(Ext
)/sizeof(Ext
[0]));
276 // PackagesWriter::PackagesWriter - Constructor /*{{{*/
277 // ---------------------------------------------------------------------
279 PackagesWriter::PackagesWriter(string DB
,string Overrides
) :
280 Db(DB
),Stats(Db
.Stats
)
287 // Process the command line options
288 DoMD5
= _config
->FindB("APT::FTPArchive::MD5",true);
289 DoContents
= _config
->FindB("APT::FTPArchive::Contents",true);
290 NoOverride
= _config
->FindB("APT::FTPArchive::NoOverrideMsg",false);
292 if (Db
.Loaded() == false)
295 // Read the override file
296 if (Overrides
.empty() == false && Over
.ReadOverride(Overrides
) == false)
300 _error
->DumpErrors();
303 // PackagesWriter::DoPackage - Process a single package /*{{{*/
304 // ---------------------------------------------------------------------
305 /* This method takes a package and gets its control information and
306 MD5 then writes out a control record with the proper fields rewritten
307 and the path/size/hash appended. */
308 bool PackagesWriter::DoPackage(string FileName
)
311 FileFd
F(FileName
,FileFd::ReadOnly
);
312 if (_error
->PendingError() == true)
315 // Stat the file for later
317 if (fstat(F
.Fd(),&St
) != 0)
318 return _error
->Errno("fstat","Failed to stat %s",FileName
.c_str());
320 // Pull all the data we need form the DB
322 if (Db
.SetFile(FileName
,St
,&F
) == false ||
323 Db
.LoadControl() == false ||
324 (DoContents
== true && Db
.LoadContents(true) == false) ||
325 (DoMD5
== true && Db
.GetMD5(MD5Res
,false) == false))
328 if (Delink(FileName
,OriginalPath
,Stats
.DeLinkBytes
,St
) == false)
331 // Lookup the overide information
332 pkgTagSection
&Tags
= Db
.Control
.Section
;
333 string Package
= Tags
.FindS("Package");
335 Override::Item
*OverItem
= Over
.GetItem(Package
);
337 if (Package
.empty() == true)
338 return _error
->Error("Archive had no package field");
340 // If we need to do any rewriting of the header do it now..
343 if (NoOverride
== false)
346 c1out
<< " " << Package
<< " has no override entry" << endl
;
350 Tmp
.Section
= Tags
.FindS("Section");
351 Tmp
.Priority
= Tags
.FindS("Priority");
355 sprintf(Size
,"%lu",St
.st_size
);
357 // Strip the DirStrip prefix from the FileName and add the PathPrefix
359 if (DirStrip
.empty() == false &&
360 FileName
.length() > DirStrip
.length() &&
361 stringcmp(FileName
.begin(),FileName
.begin() + DirStrip
.length(),
362 DirStrip
.begin(),DirStrip
.end()) == 0)
363 NewFileName
= string(FileName
.begin() + DirStrip
.length(),FileName
.end());
365 NewFileName
= FileName
;
366 if (PathPrefix
.empty() == false)
367 NewFileName
= flCombine(PathPrefix
,NewFileName
);
369 // This lists all the changes to the fields we are going to make.
370 TFRewriteData Changes
[] = {{"Size",Size
},
371 {"MD5sum",MD5Res
.c_str()},
372 {"Filename",NewFileName
.c_str()},
373 {"Section",OverItem
->Section
.c_str()},
374 {"Priority",OverItem
->Priority
.c_str()},
377 {}, // For maintainer
380 unsigned int End
= 0;
381 for (End
= 0; Changes
[End
].Tag
!= 0; End
++);
383 // Rewrite the maintainer field if necessary
385 string NewMaint
= OverItem
->SwapMaint(Tags
.FindS("Maintainer"),MaintFailed
);
386 if (MaintFailed
== true)
388 if (NoOverride
== false)
391 c1out
<< " " << Package
<< " maintainer is " <<
392 Tags
.FindS("Maintainer") << " not " <<
393 OverItem
->OldMaint
<< endl
;
397 if (NewMaint
.empty() == false)
399 Changes
[End
].Rewrite
= NewMaint
.c_str();
400 Changes
[End
++].Tag
= "Maintainer";
403 /* Get rid of the Optional tag. This is an ugly, ugly, ugly hack that
404 dpkg-scanpackages does.. Well sort of. dpkg-scanpackages just does renaming
405 but dpkg does this append bit. So we do the append bit, at least that way the
406 status file and package file will remain similar. There are other transforms
407 but optional is the only legacy one still in use for some lazy reason. */
408 string OptionalStr
= Tags
.FindS("Optional");
409 if (OptionalStr
.empty() == false)
411 if (Tags
.FindS("Suggests").empty() == false)
412 OptionalStr
= Tags
.FindS("Suggests") + ", " + OptionalStr
;
413 Changes
[End
].Rewrite
= OptionalStr
.c_str();
414 Changes
[End
++].Tag
= "Suggests";
417 // Rewrite and store the fields.
418 if (TFRewrite(Output
,Tags
,TFRewritePackageOrder
,Changes
) == false)
420 fprintf(Output
,"\n");
426 // SourcesWriter::SourcesWriter - Constructor /*{{{*/
427 // ---------------------------------------------------------------------
429 SourcesWriter::SourcesWriter(string BOverrides
,string SOverrides
)
438 // Process the command line options
439 NoOverride
= _config
->FindB("APT::FTPArchive::NoOverrideMsg",false);
441 // Read the override file
442 if (BOverrides
.empty() == false && BOver
.ReadOverride(BOverrides
) == false)
447 if (SOverrides
.empty() == false && FileExists(SOverrides
) == true &&
448 SOver
.ReadOverride(SOverrides
,true) == false)
450 // _error->DumpErrors();
453 // SourcesWriter::DoPackage - Process a single package /*{{{*/
454 // ---------------------------------------------------------------------
456 bool SourcesWriter::DoPackage(string FileName
)
459 FileFd
F(FileName
,FileFd::ReadOnly
);
460 if (_error
->PendingError() == true)
463 // Stat the file for later
465 if (fstat(F
.Fd(),&St
) != 0)
466 return _error
->Errno("fstat","Failed to stat %s",FileName
.c_str());
468 if (St
.st_size
> 128*1024)
469 return _error
->Error("DSC file '%s' is too large!",FileName
.c_str());
471 if (BufSize
< (unsigned)St
.st_size
+1)
473 BufSize
= St
.st_size
+1;
474 Buffer
= (char *)realloc(Buffer
,St
.st_size
+1);
477 if (F
.Read(Buffer
,St
.st_size
) == false)
481 char *Start
= Buffer
;
482 char *BlkEnd
= Buffer
+ St
.st_size
;
484 MD5
.Add((unsigned char *)Start
,BlkEnd
- Start
);
486 // Add an extra \n to the end, just in case
489 /* Remove the PGP trailer. Some .dsc's have this without a blank line
491 const char *Key
= "-----BEGIN PGP SIGNATURE-----";
492 for (char *MsgEnd
= Start
; MsgEnd
< BlkEnd
- strlen(Key
) -1; MsgEnd
++)
494 if (*MsgEnd
== '\n' && strncmp(MsgEnd
+1,Key
,strlen(Key
)) == 0)
501 /* Read records until we locate the Source record. This neatly skips the
502 GPG header (which is RFC822 formed) without any trouble. */
507 if (Tags
.Scan(Start
,BlkEnd
- Start
) == false)
508 return _error
->Error("Could not find a record in the DSC '%s'",FileName
.c_str());
509 if (Tags
.Find("Source",Pos
) == true)
511 Start
+= Tags
.size();
516 // Lookup the overide information, finding first the best priority.
519 string Bins
= Tags
.FindS("Binary");
520 Override::Item
*OverItem
= 0;
521 if (Bins
.empty() == false && Bins
.length() < sizeof(Buffer
))
523 strcpy(Buffer
,Bins
.c_str());
525 // Ignore too-long errors.
527 TokSplitString(',',Buffer
,BinList
,sizeof(BinList
)/sizeof(BinList
[0]));
529 // Look at all the binaries
530 unsigned char BestPrioV
= pkgCache::State::Extra
;
531 for (unsigned I
= 0; BinList
[I
] != 0; I
++)
533 Override::Item
*Itm
= BOver
.GetItem(BinList
[I
]);
539 unsigned char NewPrioV
= debListParser::GetPrio(Itm
->Priority
);
540 if (NewPrioV
< BestPrioV
|| BestPrio
.empty() == true)
542 BestPrioV
= NewPrioV
;
543 BestPrio
= Itm
->Priority
;
548 // If we need to do any rewriting of the header do it now..
552 if (NoOverride
== false)
555 c1out
<< " " << Tags
.FindS("Source") << " has no override entry" << endl
;
561 Override::Item
*SOverItem
= SOver
.GetItem(Tags
.FindS("Source"));
564 SOverItem
= BOver
.GetItem(Tags
.FindS("Source"));
566 SOverItem
= OverItem
;
569 // Add the dsc to the files hash list
571 snprintf(Files
,sizeof(Files
),"\n %s %lu %s\n %s",
572 string(MD5
.Result()).c_str(),St
.st_size
,
573 flNotDir(FileName
).c_str(),
574 Tags
.FindS("Files").c_str());
576 // Strip the DirStrip prefix from the FileName and add the PathPrefix
578 if (DirStrip
.empty() == false &&
579 FileName
.length() > DirStrip
.length() &&
580 stringcmp(DirStrip
,OriginalPath
,OriginalPath
+ DirStrip
.length()) == 0)
581 NewFileName
= string(OriginalPath
+ DirStrip
.length());
583 NewFileName
= OriginalPath
;
584 if (PathPrefix
.empty() == false)
585 NewFileName
= flCombine(PathPrefix
,NewFileName
);
587 string Directory
= flNotFile(OriginalPath
);
588 string Package
= Tags
.FindS("Source");
590 // Perform the delinking operation over all of the files
592 const char *C
= Files
;
593 for (;isspace(*C
); C
++);
596 // Parse each of the elements
597 if (ParseQuoteWord(C
,ParseJnk
) == false ||
598 ParseQuoteWord(C
,ParseJnk
) == false ||
599 ParseQuoteWord(C
,ParseJnk
) == false)
600 return _error
->Error("Error parsing file record");
603 string OriginalPath
= Directory
+ ParseJnk
;
604 if (RealPath
!= 0 && readlink(OriginalPath
.c_str(),Jnk
,sizeof(Jnk
)) != -1 &&
605 realpath(OriginalPath
.c_str(),RealPath
) != 0)
607 string RP
= RealPath
;
608 if (Delink(RP
,OriginalPath
.c_str(),Stats
.DeLinkBytes
,St
) == false)
613 Directory
= flNotFile(NewFileName
);
614 if (Directory
.length() > 2)
615 Directory
.erase(Directory
.end()-1);
617 // This lists all the changes to the fields we are going to make.
618 TFRewriteData Changes
[] = {{"Source",Package
.c_str(),"Package"},
620 {"Directory",Directory
.c_str()},
621 {"Section",SOverItem
->Section
.c_str()},
622 {"Priority",BestPrio
.c_str()},
624 {}, // For maintainer
626 unsigned int End
= 0;
627 for (End
= 0; Changes
[End
].Tag
!= 0; End
++);
629 // Rewrite the maintainer field if necessary
631 string NewMaint
= OverItem
->SwapMaint(Tags
.FindS("Maintainer"),MaintFailed
);
632 if (MaintFailed
== true)
634 if (NoOverride
== false)
637 c1out
<< " " << Package
<< " maintainer is " <<
638 Tags
.FindS("Maintainer") << " not " <<
639 OverItem
->OldMaint
<< endl
;
642 if (NewMaint
.empty() == false)
644 Changes
[End
].Rewrite
= NewMaint
.c_str();
645 Changes
[End
++].Tag
= "Maintainer";
648 // Rewrite and store the fields.
649 if (TFRewrite(Output
,Tags
,TFRewriteSourceOrder
,Changes
) == false)
651 fprintf(Output
,"\n");
659 // ContentsWriter::ContentsWriter - Constructor /*{{{*/
660 // ---------------------------------------------------------------------
662 ContentsWriter::ContentsWriter(string DB
) :
663 Db(DB
), Stats(Db
.Stats
)
671 // ContentsWriter::DoPackage - Process a single package /*{{{*/
672 // ---------------------------------------------------------------------
673 /* If Package is the empty string the control record will be parsed to
674 determine what the package name is. */
675 bool ContentsWriter::DoPackage(string FileName
,string Package
)
678 FileFd
F(FileName
,FileFd::ReadOnly
);
679 if (_error
->PendingError() == true)
682 // Stat the file for later
684 if (fstat(F
.Fd(),&St
) != 0)
685 return _error
->Errno("fstat","Failed too stat %s",FileName
.c_str());
688 if (Db
.SetFile(FileName
,St
,&F
) == false ||
689 Db
.LoadContents(false) == false)
692 // Parse the package name
693 if (Package
.empty() == true)
695 if (Db
.LoadControl() == false)
697 Package
= Db
.Control
.Section
.FindS("Package");
700 Db
.Contents
.Add(Gen
,Package
);
705 // ContentsWriter::ReadFromPkgs - Read from a packages file /*{{{*/
706 // ---------------------------------------------------------------------
708 bool ContentsWriter::ReadFromPkgs(string PkgFile
,string PkgCompress
)
710 MultiCompress
Pkgs(PkgFile
,PkgCompress
,0,false);
711 if (_error
->PendingError() == true)
714 // Open the package file
717 if (Pkgs
.OpenOld(CompFd
,Proc
) == false)
721 FileFd
Fd(CompFd
,false);
722 pkgTagFile
Tags(&Fd
);
723 if (_error
->PendingError() == true)
725 Pkgs
.CloseOld(CompFd
,Proc
);
730 pkgTagSection Section
;
731 while (Tags
.Step(Section
) == true)
733 string File
= flCombine(Prefix
,Section
.FindS("FileName"));
734 string Package
= Section
.FindS("Section");
735 if (Package
.empty() == false && Package
.end()[-1] != '/')
738 Package
+= Section
.FindS("Package");
741 Package
+= Section
.FindS("Package");
743 DoPackage(File
,Package
);
744 if (_error
->empty() == false)
746 _error
->Error("Errors apply to file '%s'",File
.c_str());
747 _error
->DumpErrors();
751 // Tidy the compressor
752 if (Pkgs
.CloseOld(CompFd
,Proc
) == false)